adminandClaude Opus 5.5
804a97b0a9
Wired into the time of Set sound FX parameters, a change then spans exactly one run of the clip, so a fade or a filter sweep ends with its last sample. - Play sound has a second output, length, in seconds. It is 0 where nothing was played or the sound has not loaded yet. - The AudioSink gives the length of a file, the SoundLibrary takes it from the decoded buffer. A pitch the voice starts on keeps it (#312). - A pitch changed while the voice plays still changes its length, so a pitch sweep over length does not end with the sound. A pitch that keeps the length in real time is #317. Tests: - soundfx, sounds, keep-on-scene-load, node-catalog, sound-library: 462 tests green. - Counter-check: with length always 0, the catalog case and the new soundfx case fail. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
514 lines
20 KiB
TypeScript
514 lines
20 KiB
TypeScript
import {
|
||
DRY_VOICE,
|
||
fxChainOf,
|
||
type FxChain,
|
||
type FxParams,
|
||
type FxStage,
|
||
type FxVoice,
|
||
} from '../core/soundfx/nodes';
|
||
import type { EaseShare } from '../core/model/animation';
|
||
import { pitchShift } from '../core/soundfx/pitchShift';
|
||
import type { AudioSink } from '../runtime/systems/Audio';
|
||
|
||
const DEFAULT_BASE = '/game-maker/data/sounds';
|
||
/** How many pitched copies are kept at once. */
|
||
const SHIFTED_KEPT = 32;
|
||
/** Seconds a changed parameter takes to get most of the way there: quick, but no click (#315). */
|
||
const GLIDE = 0.03;
|
||
/** The rate a sound loaded ahead of its first play is decoded at; a source resamples it on the way out. */
|
||
const DECODE_RATE = 48000;
|
||
|
||
/**
|
||
* One number of a voice that can still change while it plays (#315): a level, a share of a Mix, the
|
||
* gain of a band – or the pitch of a source, which glides by speed from the pitch its copy was made at.
|
||
* `even` says what a timed change runs evenly in: a share or a band in what it holds, a gain stage in
|
||
* decibels (its node holds the factor), a pitch in semitones (its node holds the speed).
|
||
*/
|
||
type Knob =
|
||
| { kind: 'level'; param: AudioParam; target: number; even: 'value' | 'ratio' }
|
||
| { kind: 'pitch'; source: AudioBufferSourceNode; base: number; target: number };
|
||
|
||
/**
|
||
* One sound in flight: its sources, its own gain, the file it came from and who is making it. A
|
||
* voice has a source per way through its sound FX (#316), started together and as long as each
|
||
* other. Its knobs are in the order `levelsOf` lists the numbers of its chain.
|
||
*/
|
||
interface Voice {
|
||
/** What `play` returned for it, the handle a `tune` reaches it by. */
|
||
id: number;
|
||
file: string;
|
||
/** The behavior this voice belongs to; `''` for one nothing owns (see `AudioSink.play`). */
|
||
owner: string;
|
||
sources: AudioBufferSourceNode[];
|
||
gain: GainNode;
|
||
fx: FxVoice;
|
||
knobs: Knob[];
|
||
}
|
||
|
||
/** What one voice is wired from: the file, and what has been made for it so far. */
|
||
interface Wiring {
|
||
context: AudioContext;
|
||
file: string;
|
||
buffer: AudioBuffer;
|
||
loop: boolean;
|
||
sources: { source: AudioBufferSourceNode; pitch: number }[];
|
||
knobs: Knob[];
|
||
stages: number;
|
||
}
|
||
|
||
/**
|
||
* A play still waiting on its bytes: what a quick `tune` in the meantime writes into, and the timed
|
||
* ones, which have to start with the voice rather than run out before it is heard.
|
||
*/
|
||
interface Pending {
|
||
id: number;
|
||
fx: FxVoice;
|
||
timed: { params: FxParams; time: number; ease: EaseShare }[];
|
||
}
|
||
|
||
/**
|
||
* Plays uploaded sounds, keyed by file name – the texture library's twin on the audio side. Files
|
||
* are content addressed, so a decoded buffer can never go stale and nothing is evicted.
|
||
*
|
||
* Everything here is asynchronous on the way in and synchronous on the way out: `play` returns at
|
||
* once and starts the voice when the bytes have been fetched and decoded. A sound fired in the
|
||
* first frame of a run is therefore heard a moment later rather than not at all, and a graph never
|
||
* waits on the network.
|
||
*
|
||
* The context is created on the first play, not at start-up: a browser refuses one before the page
|
||
* has been interacted with, and an editor that was only ever looked at should not be holding an
|
||
* audio device open. What `preload` fetches before that is decoded by an offline context, which
|
||
* holds no device.
|
||
*/
|
||
export class SoundLibrary implements AudioSink {
|
||
private base = DEFAULT_BASE;
|
||
private context: AudioContext | null = null;
|
||
private offline: OfflineAudioContext | null = null;
|
||
private readonly buffers = new Map<string, AudioBuffer>();
|
||
/** Pitched copies of decoded sounds, oldest use first (see `pitched`). */
|
||
private readonly shifted = new Map<string, AudioBuffer>();
|
||
private readonly loading = new Map<string, Promise<AudioBuffer | null>>();
|
||
/** Never retried, so one broken reference cannot spam the network. */
|
||
private readonly failed = new Set<string>();
|
||
private voices: Voice[] = [];
|
||
private pending: Pending[] = [];
|
||
/** The number the next play is given; 0 stays free for "nothing plays". */
|
||
private nextId = 1;
|
||
private started = 0;
|
||
private stopped = 0;
|
||
private tuned = 0;
|
||
/** The last voice actually started, with the values it was started at – what a check reads back. */
|
||
private last: LastVoice | null = null;
|
||
/**
|
||
* How often a stop has reached each file, and how often one has silenced everything. Counters
|
||
* rather than a "cancelled" flag: what a play has to know is whether a stop happened *after it
|
||
* was asked for*, and a flag cannot tell that from one that happened long before – which would
|
||
* make the first play of a sound disappear whenever anything had ever stopped it.
|
||
*/
|
||
private readonly stops = new Map<string, number>();
|
||
private allStops = 0;
|
||
/** The same counter per owner, for the same reason – a behavior destroyed while bytes were in
|
||
* flight must not be given its voice afterwards. */
|
||
private readonly ownerStops = new Map<string, number>();
|
||
|
||
/** Where the files are served from. Set once at start-up, before anything plays. */
|
||
configure(base: string): void {
|
||
this.base = base.replace(/\/+$/, '');
|
||
}
|
||
|
||
/**
|
||
* The number is handed out at once, before the bytes are there: a graph can tune a voice in the
|
||
* same frame it asked for it, and the voice then starts with those values.
|
||
*/
|
||
play(file: string, volume: number, loop: boolean, owner = '', fx: FxVoice = DRY_VOICE): number {
|
||
if (file === '' || this.failed.has(file)) return 0;
|
||
const id = this.nextId++;
|
||
const buffer = this.buffers.get(file);
|
||
if (buffer) {
|
||
this.start(id, file, buffer, volume, loop, owner, fx);
|
||
return id;
|
||
}
|
||
const askedAt = this.cancelsFor(file, owner);
|
||
const waiting: Pending = { id, fx: { plan: fx.plan, params: { ...fx.params } }, timed: [] };
|
||
this.pending.push(waiting);
|
||
void this.load(file).then((loaded) => {
|
||
this.pending = this.pending.filter((other) => other !== waiting);
|
||
// A `Stop sound` – or the destruction of the behavior that asked – that happened while the
|
||
// bytes were in flight has to win, or a sound the run already ended would start afterwards.
|
||
if (!loaded || this.cancelsFor(file, owner) !== askedAt) return;
|
||
this.start(id, file, loaded, volume, loop, owner, waiting.fx);
|
||
for (const { params, time, ease } of waiting.timed) this.tune(id, params, time, ease);
|
||
});
|
||
return id;
|
||
}
|
||
|
||
/** Says whether the voice was still there to be reached – playing, or waiting on its bytes. */
|
||
tune(id: number, params: FxParams, time = 0, ease: EaseShare = 'linear'): boolean {
|
||
const waiting = this.pending.find((other) => other.id === id);
|
||
if (waiting) {
|
||
// A quick change is where the voice starts; a timed one – and anything after it, so the order
|
||
// holds – runs from the moment it is heard.
|
||
if (time > 0 || waiting.timed.length > 0) waiting.timed.push({ params, time, ease });
|
||
else waiting.fx = { plan: waiting.fx.plan, params: { ...waiting.fx.params, ...params } };
|
||
return true;
|
||
}
|
||
const voice = this.voices.find((other) => other.id === id);
|
||
if (!voice) return false;
|
||
voice.fx = { plan: voice.fx.plan, params: { ...voice.fx.params, ...params } };
|
||
this.glide(voice, time, ease);
|
||
this.tuned += 1;
|
||
return true;
|
||
}
|
||
|
||
stop(file: string): void {
|
||
this.silence(file === '' ? this.voices : this.voices.filter((voice) => voice.file === file));
|
||
if (file === '') this.allStops += 1;
|
||
else this.stops.set(file, (this.stops.get(file) ?? 0) + 1);
|
||
}
|
||
|
||
/**
|
||
* Every voice of one behavior. An empty owner is deliberately no one: those are the voices that
|
||
* were started to outlive their behavior, so "stop the owner" must not reach them.
|
||
*/
|
||
stopOwner(owner: string): void {
|
||
if (owner === '') return;
|
||
this.silence(this.voices.filter((voice) => voice.owner === owner));
|
||
this.ownerStops.set(owner, (this.ownerStops.get(owner) ?? 0) + 1);
|
||
}
|
||
|
||
/** Every voice at once: what ending a run does, so nothing outlives the world that started it. */
|
||
stopAll(): void {
|
||
this.stop('');
|
||
}
|
||
|
||
/** A pause holds the sounds where they are; resuming carries them on mid-note. */
|
||
setSuspended(suspended: boolean): void {
|
||
const context = this.context;
|
||
if (!context) return;
|
||
if (suspended) void context.suspend();
|
||
else void context.resume();
|
||
}
|
||
|
||
/** What has been played and what is playing – the only witness a muted check script has. */
|
||
stats(): {
|
||
playing: number;
|
||
started: number;
|
||
stopped: number;
|
||
tuned: number;
|
||
decoded: number;
|
||
failed: number;
|
||
last: LastVoice | null;
|
||
} {
|
||
return {
|
||
playing: this.voices.length,
|
||
started: this.started,
|
||
stopped: this.stopped,
|
||
tuned: this.tuned,
|
||
decoded: this.buffers.size,
|
||
failed: this.failed.size,
|
||
last: this.last,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Where the knobs of each voice of that file are right now, oldest first, a pitch as the speed it
|
||
* plays at – what a check reads to see a change arrive at one voice and not another (test hook).
|
||
*/
|
||
levels(file: string): number[][] {
|
||
return this.voices
|
||
.filter((voice) => voice.file === file)
|
||
.map((voice) =>
|
||
voice.knobs.map((knob) => (knob.kind === 'pitch' ? knob.source.playbackRate : knob.param).value),
|
||
);
|
||
}
|
||
|
||
/** How long a decoded sound runs, in seconds; null while it has not been loaded (test hook). */
|
||
duration(file: string): number | null {
|
||
return this.buffers.get(file)?.duration ?? null;
|
||
}
|
||
|
||
length(file: string): number {
|
||
return this.duration(file) ?? 0;
|
||
}
|
||
|
||
/**
|
||
* Starts loading every file not held yet. Waiting for the first play left that play late: its
|
||
* start was not heard, and a graph tuning it in the meantime had moved on by the time it started.
|
||
*/
|
||
preload(files: Iterable<string>): void {
|
||
for (const file of files) if (file) void this.load(file);
|
||
}
|
||
|
||
/** Fetches and decodes once per file; a second play while it is in flight waits on the first. */
|
||
async load(file: string): Promise<AudioBuffer | null> {
|
||
const held = this.buffers.get(file);
|
||
if (held) return held;
|
||
if (this.failed.has(file)) return null;
|
||
const running = this.loading.get(file);
|
||
if (running) return running;
|
||
const work = (async () => {
|
||
try {
|
||
const response = await fetch(`${this.base}/${file}`);
|
||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||
const bytes = await response.arrayBuffer();
|
||
const buffer = await this.decoder().decodeAudioData(bytes);
|
||
this.buffers.set(file, buffer);
|
||
return buffer;
|
||
} catch (error: unknown) {
|
||
this.failed.add(file);
|
||
console.error(`[sounds] cannot load ${file}`, error);
|
||
return null;
|
||
} finally {
|
||
this.loading.delete(file);
|
||
}
|
||
})();
|
||
this.loading.set(file, work);
|
||
return work;
|
||
}
|
||
|
||
/** Ends these voices now: `onended` only fires next frame and the counts have to be right here. */
|
||
private silence(going: Voice[]): void {
|
||
for (const voice of going) for (const source of voice.sources) source.stop();
|
||
this.voices = this.voices.filter((voice) => !going.includes(voice));
|
||
this.stopped += going.length;
|
||
}
|
||
|
||
private start(
|
||
id: number,
|
||
file: string,
|
||
buffer: AudioBuffer,
|
||
volume: number,
|
||
loop: boolean,
|
||
owner: string,
|
||
fx: FxVoice,
|
||
): void {
|
||
const context = this.audio();
|
||
// A context starts suspended where the page has not been interacted with; without this the
|
||
// voice would run silently and end as if it had played.
|
||
if (context.state === 'suspended') void context.resume();
|
||
const gain = context.createGain();
|
||
gain.gain.value = Math.max(0, Math.min(1, volume));
|
||
const wiring: Wiring = { context, file, buffer, loop, sources: [], knobs: [], stages: 0 };
|
||
this.wire(wiring, fxChainOf(fx.plan, fx.params)).connect(gain).connect(context.destination);
|
||
const sources = wiring.sources.map((made) => made.source);
|
||
this.last = {
|
||
file,
|
||
volume: gain.gain.value,
|
||
loop,
|
||
pitches: wiring.sources.map((made) => made.pitch),
|
||
stages: wiring.stages,
|
||
duration: sources[0]!.buffer!.duration,
|
||
};
|
||
const voice: Voice = { id, file, owner, sources, gain, fx, knobs: wiring.knobs };
|
||
let playing = sources.length;
|
||
for (const source of sources)
|
||
source.onended = () => {
|
||
if (--playing === 0) this.voices = this.voices.filter((other) => other !== voice);
|
||
};
|
||
this.voices.push(voice);
|
||
this.started += 1;
|
||
for (const source of sources) source.start();
|
||
}
|
||
|
||
/**
|
||
* The nodes a chain becomes, up to the one that carries its sound on. Every way has a source of
|
||
* its own, so each can glide to its own pitch; a Mix weighs each of its sounds by its share before
|
||
* they are summed.
|
||
*/
|
||
private wire(wiring: Wiring, chain: FxChain): AudioNode {
|
||
const { context, knobs } = wiring;
|
||
let tail: AudioNode;
|
||
if ('mix' in chain) {
|
||
tail = context.createGain();
|
||
const wayA = this.wire(wiring, chain.a);
|
||
const shareA = gainNode(context, 1 - chain.mix);
|
||
wayA.connect(shareA).connect(tail);
|
||
knobs.push({ kind: 'level', param: shareA.gain, target: 1 - chain.mix, even: 'value' });
|
||
const wayB = this.wire(wiring, chain.b);
|
||
const shareB = gainNode(context, chain.mix);
|
||
wayB.connect(shareB).connect(tail);
|
||
knobs.push({ kind: 'level', param: shareB.gain, target: chain.mix, even: 'value' });
|
||
} else tail = this.source(wiring, chain.pitch);
|
||
for (const stage of chain.stages) {
|
||
const [node, param] = fxNode(context, stage);
|
||
tail = tail.connect(node);
|
||
knobs.push({
|
||
kind: 'level',
|
||
param,
|
||
target: stage.gain,
|
||
even: stage.kind === 'gain' ? 'ratio' : 'value',
|
||
});
|
||
}
|
||
wiring.stages += chain.stages.length;
|
||
return tail;
|
||
}
|
||
|
||
private source(wiring: Wiring, pitch: number): AudioBufferSourceNode {
|
||
const { context, file, buffer, loop } = wiring;
|
||
const source = context.createBufferSource();
|
||
source.buffer = pitch === 0 ? buffer : this.pitched(context, file, buffer, pitch, loop);
|
||
source.loop = loop;
|
||
wiring.sources.push({ source, pitch });
|
||
wiring.knobs.push({ kind: 'pitch', source, base: pitch, target: pitch });
|
||
return source;
|
||
}
|
||
|
||
/**
|
||
* Sends every knob of a voice towards what its values say now, in `time` seconds of the audio
|
||
* clock along `ease`, or else in a quick glide. A pitch cannot get a new copy in the middle of a sound, so it
|
||
* glides by speed: the voice gets that much shorter or longer.
|
||
*/
|
||
private glide(voice: Voice, time: number, ease: EaseShare): void {
|
||
const levels = levelsOf(fxChainOf(voice.fx.plan, voice.fx.params));
|
||
const now = this.audio().currentTime;
|
||
voice.knobs.forEach((knob, i) => {
|
||
const level = levels[i];
|
||
if (level === undefined || level === knob.target) return;
|
||
knob.target = level;
|
||
if (knob.kind === 'pitch')
|
||
move(knob.source.playbackRate, 2 ** ((level - knob.base) / 12), now, time, true, ease);
|
||
else move(knob.param, level, now, time, knob.even === 'ratio', ease);
|
||
});
|
||
}
|
||
|
||
/**
|
||
* The sound at another pitch and its own length (#312), worked out once per file, pitch and loop
|
||
* and kept for the next voice. The oldest goes first, so a pitch drawn at random every play cannot
|
||
* fill the memory.
|
||
*/
|
||
private pitched(
|
||
context: AudioContext,
|
||
file: string,
|
||
buffer: AudioBuffer,
|
||
pitch: number,
|
||
loop: boolean,
|
||
): AudioBuffer {
|
||
const key = `${file}\u0000${pitch}\u0000${loop}`;
|
||
const held = this.shifted.get(key);
|
||
if (held) {
|
||
this.shifted.delete(key);
|
||
this.shifted.set(key, held);
|
||
return held;
|
||
}
|
||
const shifted = context.createBuffer(buffer.numberOfChannels, buffer.length, buffer.sampleRate);
|
||
for (let channel = 0; channel < buffer.numberOfChannels; channel++)
|
||
shifted
|
||
.getChannelData(channel)
|
||
.set(pitchShift(buffer.getChannelData(channel), buffer.sampleRate, pitch, loop));
|
||
this.shifted.set(key, shifted);
|
||
if (this.shifted.size > SHIFTED_KEPT) this.shifted.delete(this.shifted.keys().next().value!);
|
||
return shifted;
|
||
}
|
||
|
||
private audio(): AudioContext {
|
||
return (this.context ??= new AudioContext());
|
||
}
|
||
|
||
/** The context that is already playing, or else one that decodes without opening a device. */
|
||
private decoder(): BaseAudioContext {
|
||
if (this.context || typeof OfflineAudioContext === 'undefined') return this.audio();
|
||
return (this.offline ??= new OfflineAudioContext(1, 1, DECODE_RATE));
|
||
}
|
||
|
||
/** Everything that would have silenced this file for this owner, counted rather than flagged. */
|
||
private cancelsFor(file: string, owner: string): number {
|
||
return (this.stops.get(file) ?? 0) + this.allStops + (this.ownerStops.get(owner) ?? 0);
|
||
}
|
||
}
|
||
|
||
/** What `stats` says about the last voice: enough for a check to tell which effects it went through. */
|
||
interface LastVoice {
|
||
file: string;
|
||
volume: number;
|
||
loop: boolean;
|
||
/** Semitones of each source, one per way (#316), and how many gain and filter nodes it passed. */
|
||
pitches: number[];
|
||
stages: number;
|
||
/** Seconds of what was played: a pitched voice is as long as the sound itself (#312). */
|
||
duration: number;
|
||
}
|
||
|
||
function gainNode(context: AudioContext, level: number): GainNode {
|
||
const gain = context.createGain();
|
||
gain.gain.value = level;
|
||
return gain;
|
||
}
|
||
|
||
/** The node a stage becomes, with the number of it a parameter can move. */
|
||
function fxNode(context: AudioContext, stage: FxStage): [AudioNode, AudioParam] {
|
||
if (stage.kind === 'gain') {
|
||
const gain = gainNode(context, stage.gain);
|
||
return [gain, gain.gain];
|
||
}
|
||
const filter = context.createBiquadFilter();
|
||
filter.type = stage.filter;
|
||
filter.frequency.value = stage.frequency;
|
||
filter.gain.value = stage.gain;
|
||
filter.Q.value = stage.q;
|
||
return [filter, filter.gain];
|
||
}
|
||
|
||
/**
|
||
* Sends one knob from where it is right now to `to`. Whatever was still planned for it is dropped
|
||
* first, so a change never queues behind an older one. A timed change runs along `ease` – a `ratio`
|
||
* one in its logarithm, which is what decibels and semitones are. Linear and step are the clock's
|
||
* own ramps, every other easing is handed over as a curve of samples.
|
||
*/
|
||
function move(
|
||
param: AudioParam,
|
||
to: number,
|
||
now: number,
|
||
time: number,
|
||
ratio: boolean,
|
||
ease: EaseShare = 'linear',
|
||
): void {
|
||
const from = param.value;
|
||
// Firefox has no cancelAndHoldAtTime: there the value it reads out now is held by hand.
|
||
const held = param as Omit<AudioParam, 'cancelAndHoldAtTime'> &
|
||
Partial<Pick<AudioParam, 'cancelAndHoldAtTime'>>;
|
||
if (held.cancelAndHoldAtTime) held.cancelAndHoldAtTime(now);
|
||
else {
|
||
param.cancelScheduledValues(now);
|
||
param.setValueAtTime(from, now);
|
||
}
|
||
const logarithmic = ratio && from > 0 && to > 0;
|
||
if (time <= 0) param.setTargetAtTime(to, now, GLIDE);
|
||
else if (ease === 'step') param.setValueAtTime(to, now + time);
|
||
else if (ease === 'linear' && logarithmic) param.exponentialRampToValueAtTime(to, now + time);
|
||
else if (ease === 'linear') param.linearRampToValueAtTime(to, now + time);
|
||
else param.setValueCurveAtTime(easedCurve(from, to, time, logarithmic, ease), now, time);
|
||
}
|
||
|
||
/** Samples per second of an eased curve: a bounce keeps its corners, a long sweep stays small. */
|
||
const CURVE_RATE = 200;
|
||
|
||
function easedCurve(
|
||
from: number,
|
||
to: number,
|
||
time: number,
|
||
logarithmic: boolean,
|
||
share: (t: number) => number,
|
||
): Float32Array {
|
||
const count = Math.min(Math.max(Math.ceil(time * CURVE_RATE) + 1, 16), 4096);
|
||
const curve = new Float32Array(count);
|
||
for (let i = 0; i < count; i++) {
|
||
const s = share(i / (count - 1));
|
||
curve[i] = logarithmic ? from * (to / from) ** s : from + (to - from) * s;
|
||
}
|
||
return curve;
|
||
}
|
||
|
||
/** The numbers of a chain in the order `wire` makes their knobs: a stage's `gain` is what its node holds. */
|
||
function levelsOf(chain: FxChain, into: number[] = []): number[] {
|
||
if ('mix' in chain) {
|
||
levelsOf(chain.a, into);
|
||
into.push(1 - chain.mix);
|
||
levelsOf(chain.b, into);
|
||
into.push(chain.mix);
|
||
} else into.push(chain.pitch);
|
||
for (const stage of chain.stages) into.push(stage.gain);
|
||
return into;
|
||
}
|
||
|
||
export const soundLibrary = new SoundLibrary();
|