import { DRY_VOICE, fxChainOf, fxLatencyOf, type FxChain, type FxChip, type FxEnvelope, fxRings, fxTailOf, impulsesOf, type FxParams, type FxPlan, type FxStagePlan, type FxVoice, stageLatency, } from '../core/soundfx/nodes'; import { releaseTime } from '../core/instrument/envelope'; import type { EaseShare } from '../core/model/animation'; import { BAKE_RATE } from '../core/music/bake'; import { type BakePlay, type BusLevel, Playbacks, type SequenceNeeds, type SequencePlay, } from '../core/music/playbacks'; import { arrivalOf, sendDelays, type TimedBus } from '../core/music/routing'; import type { SendTap } from '../core/graph/Graph'; import { MASTER_MIX, type MixBus, type MixSend, sameMix } from '../core/music/mixer'; import { limiterLatency } from '../core/soundfx/dynamics'; import type { ScheduledClip, ScheduledNote } from '../core/music/Scheduler'; import { dbToGain, SOUND_ITEM_MAX_DB } from '../core/music/song'; import { DEFAULT_TEMPO } from '../core/music/time'; import { playedPlan, type StreamedFile, type StreamPlan } from '../core/music/streams'; import { STREAM_RATE } from '../core/music/streamQueue'; import { type CutRange, cutChannels, cutRange, parseCutName } from '../core/model/sound'; import { shifterLatency } from '../core/soundfx/pitchPlayer'; import { REVERB_MAX_DECAY } from '../core/soundfx/reverb'; import type { AudioSink, ClipTiming, NoteTiming } from '../runtime/systems/Audio'; import type { ChipOptions, PitchOptions, StreamOptions } from './fxWorklet'; import fxWorkletUrl from './fxWorklet.ts?worker&url'; import { bothAtCenter, endedProcessor, envelopeDriver, FADE, type Follow, followCurve, fadeOut, gainNode, GLIDE, glideKnobs, heldBack, type Knob, levelsOf, move, stageNodes, releaseEnvelope, stopProcessor, } from './fxNodes'; import { type Levels, type Spectrum, SpectrumTaps } from './spectrum'; import { decodeSpan, StreamFeed, StreamFile } from './streaming'; import { wake } from './wake'; /** A voice is no louder than a sound item of a song turned all the way up. */ const MAX_VOLUME = dbToGain(SOUND_ITEM_MAX_DB); const DEFAULT_BASE = '/game-maker/data/sounds'; /** The rate a sound loaded ahead of its first play is decoded at; a source resamples it on the way out. */ const DECODE_RATE = 48000; /** The rate the picture of a streamed sound is decoded at (#349): enough for its peaks, a sixth of the size. */ const PICTURE_RATE = 8000; /** The longest a send of a track is held back to meet the latest way of it (#350). */ const MAX_SEND_DELAY = 1; /** Samples a level meter looks back over (#359): a frame of 40 ms at 48 kHz misses no peak. */ const METER_SPAN = 2048; /** The loudest the limiter at the output lets a sample be (#359), and how fast it lets go again. */ const GUARD_DB = -1; const GUARD_RELEASE = 0.1; /** * The most ways that play through the pitch worklet at once, a phase vocoder each (#317). Measured in * Chrome on the Windows runner: 12 take half of the audio thread, 24 all of it, and past that every * sound drops out until enough of them have ended. */ const MAX_VOCODERS = 12; /** How finely `peaks` cuts a sound up. */ const PEAK_BUCKETS = 1024; /** Samples the cuts made so far may hold together before the oldest are let go; one is always kept. */ const CUT_BUDGET = 64 * 1024 * 1024; /** * One way's sound as a voice holds it: a buffer source, or the pitch worklet where the way has a * Pitch (#317). A voice that can ring on gives each of its sources a `fade` of its own, so they can * fall silent without a click while the echoes carry on (#324). */ interface Source { /** * At that second of the audio clock (#332), `offset` seconds into the sound (#334); a worklet was * told both when it was made. */ start(at: number, offset: number): void; stop(): void; ended(then: () => void): void; fade: GainNode | null; /** Ends with the note (#331): a release fades it out, where everything else plays on. */ gate: boolean; /** Plays on past its loop to the end of the sound; null where a release leaves it as it is. */ leave: (() => void) | null; } /** Something of a voice that is started with it and stopped when it goes. */ interface Driver { start(): void; stop(): void; } /** The ways of a voice on the pitch worklet still running: none once only its echoes are left. */ const vocodersOf = (voice: Voice) => (voice.ringing ? 0 : voice.vocoders); /** Whether a note's end leaves a source as it is: a one-shot, which plays to the end of its sound. */ const playsThrough = (source: Source) => !source.gate && !source.leave; /** * One Envelope of a note (#335): what carries its curve on the audio thread, and the sources whose * volume it is – they end with its release. One that only moves numbers has none. */ interface EnvelopeRun { envelope: FxEnvelope; driver: ConstantSourceNode; sources: Source[]; released: boolean; } /** * One sound in flight: its sources, its own gain, the file it came from and who is making it. A * voice has a source per way through its sound FX (#316), started together and as long as each * other. A note of an instrument (#331) is one voice too, its layers side by side, each way with a * file of its own. Its knobs are in the order `levelsOf` lists the numbers of its chains. */ interface Voice { /** What `play` returned for it, the handle a `tune` reaches it by. */ id: number; /** The file it plays, or for a note what `playNote` was given to name its instrument by. */ file: string; /** The behavior this voice belongs to; `''` for one nothing owns (see `AudioSink.play`). */ owner: string; /** The bus it is heard on, `''` straight out. */ bus: string; /** When it started on the audio clock, what its Envelopes count from. */ at: number; sources: Source[]; /** Sources already set to end, by a release or a stop. */ ending: Set; /** * What carries a number of it on the audio thread (a Mix "Both at center", an Envelope) or runs * there until told otherwise (a Bitcrusher): stopped when it goes. */ drivers: Driver[]; envelopes: EnvelopeRun[]; /** Calls the end of one source in, once however it comes: by itself or by a release. */ finish: ((source: Source) => void) | null; gain: GainNode; fx: Layered; knobs: Knob[]; /** Whether its FX can ring on after the sources end (#324), and whether that is all it does now. */ rings: boolean; ringing: boolean; /** Calls off the clock that ends its ringing. */ quiet: (() => void) | null; /** How many of its ways play through the pitch worklet, and whether its note has been let go. */ vocoders: number; released: boolean; /** Seconds a stop that lets it ring fades it out over: a bake (#340), whose notes are one sound. */ ringFade: number; } /** * The plans of a voice and the values of their parameters: one plan for a sound played through its * FX, one per layer for a note (#331), and the tempo a Delay synced to beats counts by (#332). */ interface Layered { layers: readonly FxPlan[]; params: FxParams; tempo: number; } /** One way of a chain: what a source plays. */ type Way = Exclude; /** * What one voice is wired from: its buffers by file, the file and loop a way without a source of * its own plays (the sound a `play` was given), and what has been made for it so far. */ interface Wiring { context: BaseAudioContext; /** When its sources start on the audio clock. */ at: number; buffers: ReadonlyMap; file: string; loop: boolean; rings: boolean; /** Seconds into the sound its ways start, and whether it is a clip of a song, which fades out at its end (#334). */ offset: number; clip: boolean; /** The copy a way without a source of its own streams (#349), instead of a buffer of `file`. */ stream: StreamFile | null; sources: Source[]; drivers: Driver[]; envelopes: Map; pitches: number[]; knobs: Knob[]; stages: number; vocoders: number; } /** Everything a voice is started with, the same whether it waits on its bytes first or not. */ interface Start { id: number; file: string; /** The files its ways read, every one of them loaded before it starts. */ files: readonly string[]; volume: number; loop: boolean; owner: string; fx: Layered; /** Seconds until a note is released (#331); 0 holds it until it is released or stopped. */ duration: number; /** When it starts on the audio clock (#332); one already past starts now. */ at: number; /** * A clip of a song (#334): seconds into the sound it starts, and the second of the audio clock it * fades out at. */ clip?: { offset: number; end: number }; /** The track of a song it is heard on (#334), `''` straight out. */ bus?: string; /** The copy `file` streams from (#349); its bytes are not loaded then, its head is. */ stream?: StreamFile; } /** * A play still waiting on its bytes: what a quick `tune` in the meantime writes into, and the timed * ones, which have to start with the voice rather than run out before it is heard. */ interface Pending { id: number; bus: string; fx: Layered; timed: { params: FxParams; time: number; ease: EaseShare }[]; /** Stopped by its number before it started. */ stopped: boolean; /** A note released before it started: it starts and is released at once, so a one-shot is heard. */ released: boolean; } /** * Plays uploaded sounds, keyed by file name – the texture library's twin on the audio side. Files * are content addressed, so a decoded buffer can never go stale and nothing is evicted. * * Everything here is asynchronous on the way in and synchronous on the way out: `play` returns at * once and starts the voice when the bytes have been fetched and decoded. A sound fired in the * first frame of a run is therefore heard a moment later rather than not at all, and a graph never * waits on the network. * * The context is created on the first play, not at start-up: a browser refuses one before the page * has been interacted with, and an editor that was only ever looked at should not be holding an * audio device open. What `preload` fetches before that is decoded by an offline context, which * holds no device. * * A way with a Pitch is played by a worklet (#317), loaded into the context with the first play that * needs it; that play waits for it as it would for its bytes. * * A sound that streams (#349) is the exception to "decoded whole": a `stream-player` on that worklet * plays it from its copy, fed a few seconds ahead (`streaming.ts`), and its file is never decoded. */ export class SoundLibrary implements AudioSink { private base = DEFAULT_BASE; private context: BaseAudioContext | null = null; /** * Where the master goes out (#359): through a limiter that keeps every sample at `GUARD_DB` or * below, once the worklet is there; straight to the speakers until then. Late by the limiter. */ private guard: { input: GainNode; limiter: AudioWorkletNode | null; latency: number } | null = null; private guardMeter: BusMeter | null = null; /** * `guarded` false leaves the limiter out: for checks that follow every node of a voice. With * `render` it plays into an offline context to bake (#340), never resumed by itself. */ constructor( private readonly guarded = true, private readonly render: OfflineRender | null = null, ) { if (render) { this.context = render.context; this.held = true; } } private decodeContext: OfflineAudioContext | null = null; /** Where a bake (#340) is decoded: always at the rate it was made at, so its loop lands on the sample. */ private opusContext: OfflineAudioContext | null = null; private readonly buffers = new Map(); /** * The stretches of files that sound assets are cut to (#343), made from the whole file on first * play, oldest first. Unlike a file they can be made again at any time, so they are let go past a * budget: a cut dragged about in its tab would otherwise leave a copy of the sound per try. */ private readonly cuts = new Map(); /** The pitch worklet being loaded, and whether it is there once it is known; null before that. */ private workletLoading: Promise | null = null; private hasWorklet: boolean | null = null; /** The files whose samples the worklet holds already, so the next voice of one is sent without. */ private readonly sent = new Set(); private readonly loading = new Map>(); /** Never retried, so one broken reference cannot spam the network. */ private readonly failed = new Set(); private voices: Voice[] = []; private pending: Pending[] = []; /** The number the next play is given; 0 stays free for "nothing plays". */ private nextId = 1; private started = 0; private stopped = 0; private tuned = 0; /** Second of the audio clock the last change moved a playing voice: a timed one runs from there. */ private tunedAt: number | null = null; /** The last voice actually started, with the values it was started at – what a check reads back. */ private last: LastVoice | null = null; /** * How often a stop has reached each file, and how often one has silenced everything. Counters * rather than a "cancelled" flag: what a play has to know is whether a stop happened *after it * was asked for*, and a flag cannot tell that from one that happened long before – which would * make the first play of a sound disappear whenever anything had ever stopped it. */ private readonly stops = new Map(); private allStops = 0; /** The same counter per owner, for the same reason – a behavior destroyed while bytes were in * flight must not be given its voice afterwards. */ private readonly ownerStops = new Map(); /** The tracks of the songs played so far (#334), by name. */ private readonly buses = new Map(); /** The taps of the voices and buses playing now, and the node listened to (#374). */ private readonly spectra = new SpectrumTaps(); private readonly peaked = new Map(); private readonly pictures = new Map>(); /** * The patterns and songs a game plays, each on a scheduler of its own. What it asks after are its * own voices – asking `playing`, which counts the playings too, would ask the list it is in. */ private readonly sequences = new Playbacks({ now: () => this.time(), latency: () => this.latency(), playing: (id) => this.sounding(id), stop: (voices, letRing) => { for (const id of voices) this.stopVoice(id, letRing); }, wake: (pump) => (this.render?.wake ?? wake)(pump), prepare: (needs, id) => this.ready(needs, id), ahead: (id) => this.groupAhead(id), level: (id, gain, at, seconds) => this.level(id, gain, at, seconds), after: (at, then) => void this.after(at, then), closed: () => this.sweep(), playBaked: (id, bake, at) => this.playBaked(id, bake, at), }); /** The playings whose buses are still there: a master each, and the tracks of a song into it. */ private readonly groups = new Set(); /** Held by the pause of the toolbar: nothing that reads the clock may wake it then. */ private held = false; /** Counts the times buses were let go, so a later use of one can keep it. */ private leavings = 0; /** * The master and the buses of the project (#325) as last set, made when the context is: setting * them must not open an audio device for an editor nobody has listened to yet. */ private mix: readonly MixBus[] = []; /** The levels a running game set its buses to, over those of their assets; until `resetMix`. */ private readonly mixVolumes = new Map(); /** The names streamed (#349) and the copies they stream from, by name and by copy. */ private streams: ReadonlyMap = new Map(); /** The cut of each clip, with the copy it is decoded from rather than its whole file (#349). */ private copies: ReadonlyMap = new Map(); private readonly streamFiles = new Map(); private readonly feeds = new Set(); /** Every sample streamed so far that came too late to be heard, of the feeds gone (test hook). */ private skippedBefore = 0; /** Where the files are served from. Set once at start-up, before anything plays. */ configure(base: string): void { this.base = base.replace(/\/+$/, ''); } /** * The names a game streams rather than decodes (#349), each with its copy (`streamTable`): the first * seconds of every copy are fetched and decoded now, so it starts at once. A library that bakes, * and a browser without Opus in WebCodecs, decode everything as before. */ setStreams(table: ReadonlyMap): void { this.streams = table; for (const name of table.keys()) void this.streamed(name)?.ready(); } /** * The copies clips are decoded from (#349), each under the cut it plays (`clipCopyTable`). A clip * whose sound's whole file is decoded anyway is cut from that as before. One that bakes keeps to the * whole files, as the fingerprint of a bake does. */ setCopies(table: ReadonlyMap): void { this.copies = table; } /** The copy a clip is decoded from, while it has one that has not failed. */ private copyOf(name: string): string | null { const copy = this.render ? undefined : this.copies.get(name); return copy && !this.failed.has(copy) ? copy : null; } /** The copy a name streams from, while it streams and the copy has not failed. */ private streamed(name: string): StreamFile | null { const entry = this.streams.get(name); if (!entry || this.render || typeof AudioDecoder === 'undefined') return null; let file = this.streamFiles.get(entry.file); if (!file) { file = new StreamFile(`${this.base}/${entry.file}`, entry.frames); this.streamFiles.set(entry.file, file); } return file.failed ? null : file; } /** * The number is handed out at once, before the bytes are there: a graph can tune a voice in the * same frame it asked for it, and the voice then starts with those values. */ play(file: string, volume: number, loop: boolean, owner = '', fx: FxVoice = DRY_VOICE, bus = ''): number { if (file === '' || this.hasFailed(file)) return 0; const layered = { layers: [fx.plan], params: fx.params, tempo: fx.tempo ?? DEFAULT_TEMPO }; const stream = this.streamed(file); return this.begin({ id: 0, file, files: [file], volume, loop, owner, fx: layered, duration: 0, at: 0, bus, ...(stream && { stream }), }); } /** * One note of an instrument (#331): its layers played side by side as one voice, each way with its * own file. `name` stands for the instrument where a file would stand – what `levels` finds it by. * A `duration` above 0 releases it that many seconds after it starts. `timing.at` starts it at that * second of the audio clock, as the scheduler plans a note (#332). */ playNote( name: string, layers: readonly FxPlan[], params: FxParams, volume: number, owner = '', duration = 0, timing: NoteTiming = {}, ): number { const files = [...new Set(layers.flatMap(filesOf))].filter( (file) => file !== '' && !this.hasFailed(file), ); if (files.length === 0 && !layers.some(hasChip)) return 0; const fx = { layers, params, tempo: timing.tempo ?? DEFAULT_TEMPO }; const at = timing.at ?? 0; const bus = timing.bus ?? ''; return this.begin({ id: 0, file: name, files, volume, loop: false, owner, fx, duration, at, bus }); } /** * A sound as a clip of a song (#334): from `at` on the audio clock, `offset` seconds into it, round * and round until `duration` seconds have passed, then faded out – its echoes ring on. One that * gets its bytes late starts as far into the sound as it would be by then. */ playClip(file: string, clip: ClipTiming, fx: FxVoice = DRY_VOICE): number { if (file === '' || this.hasFailed(file)) return 0; const layered = { layers: [fx.plan], params: fx.params, tempo: fx.tempo ?? DEFAULT_TEMPO }; const { at, offset, duration, volume, bus } = clip; const stream = this.streamed(file); return this.begin({ id: 0, file, files: [file], volume, loop: true, owner: '', fx: layered, duration: 0, at, clip: { offset, end: at + Math.max(0, duration) }, bus, ...(stream && { stream }), }); } /** * Sets the level and the pan of a track of a song (#334) in a quick glide, so a change reaches * what is sounding on it already. A bus is made on its first use and kept. */ setBus(name: string, gain: number, pan: number): void { const bus = this.bus(name); const now = this.audio().currentTime; bus.set = { gain: Math.max(0, gain), pan: Math.max(-1, Math.min(1, pan)) }; bus.gain.gain.setTargetAtTime(Math.max(0, gain), now, GLIDE / 3); bus.pan.pan.setTargetAtTime(Math.max(-1, Math.min(1, pan)), now, GLIDE / 3); } /** * The tracks of a song in the Song tab (#334), each set as `setBus` sets one, with the bus it goes * into and its sound FX (#346). A chain is made again only where its FX has changed. Done once the * chains are there – the worklet some of them need has to be loaded first. */ async setBuses(levels: readonly BusLevel[]): Promise { const waits: Promise[] = []; for (const { name, gain, pan, into, sends, fx } of levels) { if (into !== undefined) this.bus(into); const bus = this.bus(name, into ?? null); if (sends) this.setSends( bus, sends.map((send) => ({ ...send, to: send.to ?? MASTER_MIX })), ); else this.route(name, bus, into ?? null); this.setBus(name, gain, pan); waits.push(this.setFx(bus, fx ?? null)); } await Promise.all(waits); } /** * Lets these buses go once what is on them now has faded (#346): their chains would otherwise run * on for nothing. One used again before then stays. */ releaseBuses(names: readonly string[]): void { const going = names.flatMap((name) => this.buses.get(name) ?? []); if (going.length === 0) return; const token = ++this.leavings; for (const bus of going) bus.leaving = token; const wait = Math.max(0, ...names.map((name) => this.ringOf(name))); this.after(this.audio().currentTime + FADE + wait, () => { for (const name of names) { const bus = this.buses.get(name); if (bus && bus.leaving === token) this.dropBus(name, bus); } }); } /** * Seconds what goes in on that bus is late by the time it is heard (#346): its chain and those of * the buses it goes into, along the latest of its sends (#350). 0 straight out. */ busLatency(name: string): number { return arrivalOf(name || MASTER_MIX, this.timed); } private readonly timed = (name: string): TimedBus | undefined => { const bus = this.buses.get(name); if (!bus) return undefined; return { latency: bus.chain?.latency ?? 0, into: bus.into, ...(bus.sends && { sends: bus.sends }) }; }; /** What each track's bus was last set to, with whether its sound FX is there and how late (test hook). */ busLevels(): { name: string; gain: number; pan: number; into: string | null; fx: boolean; latency: number; sends?: { to: string; tap: SendTap; gain: number; delay: number }[]; }[] { return [...this.buses].map(([name, bus]) => ({ name, ...bus.set, into: bus.sends ? null : bus.into, fx: bus.chain !== null, latency: bus.chain?.latency ?? 0, ...(bus.sends && { sends: bus.sends.map(({ to, tap, gain, delay }) => ({ to, tap, gain, delay: delay.delayTime.value })), }), })); } /** * The loudest sample on each side of these buses over the last moment (#359), as they leave them * after level and pan, 1 at full scale. A bus is tapped the first time it is asked after, one not * there is left out. The taps stay until `unmeter`. */ busPeaks(names: readonly string[]): Map { const found = new Map(); for (const name of names) { const bus = this.buses.get(name); if (!bus) continue; found.set(name, readMeter((bus.meter ??= this.tapMeter(bus.pan)))); } return found; } /** * The loudest sample on each side as it leaves the limiter at the output (#359), tapped the first * time it is asked (test hook); null while there is no limiter. */ guardPeaks(): [number, number] | null { const guard = this.guard; if (!guard?.limiter) return null; return readMeter((this.guardMeter ??= this.tapMeter(guard.limiter))); } /** Lets every level meter go. */ unmeter(): void { for (const bus of this.buses.values()) this.dropMeter(bus); } /** * Listens to the sound in front of the node `id` (`tapId`) and behind it (#374): the tab of a * Parametric EQ or a Compressor (#378) while it is seen. `spectrum` and `nodeLevels` read it, * `unlisten` lets it go. */ listen(id: string): void { this.spectra.listen(id); } unlisten(id: string): void { this.spectra.stop(id); } spectrum(id: string): Spectrum | null { return this.spectra.read(id); } nodeLevels(id: string, seconds: number, lateBy = 0): Levels | null { return this.spectra.levels(id, seconds, lateBy); } /** The rate the page's audio runs at, once there is any; null before. */ sampleRate(): number | null { return this.context?.sampleRate ?? null; } /** The node listened to, the taps playing, and how many of them feed an analyser (test hook). */ spectrumTaps(): { listened: string; held: number; feeding: number } { return this.spectra.info(); } /** How many buses are tapped for a meter (test hook). */ metered(): number { return [...this.buses.values()].filter((bus) => bus.meter).length; } private tapMeter(from: AudioNode): BusMeter { const context = this.audio(); const split = context.createChannelSplitter(2); const sides = [0, 1].map((channel) => { const side = context.createAnalyser(); side.fftSize = METER_SPAN; split.connect(side, channel); return side; }); from.connect(split); return { split, sides, samples: new Float32Array(METER_SPAN) }; } private dropMeter(bus: Bus): void { const meter = bus.meter; if (!meter) return; bus.meter = null; try { bus.pan.disconnect(meter.split); } catch { // Already cut with the rest of the bus. } meter.split.disconnect(); } /** Cuts the pan of a bus from where it went; its meter stays on. */ private cutPan(bus: Bus): void { bus.pan.disconnect(); if (bus.meter) bus.pan.connect(bus.meter.split); } /** Where the voices of a bus go in; the master for `''` (#325). */ private busInput(name: string): AudioNode { return this.bus(name || MASTER_MIX).input; } /** * A bus by its name, made on its first use into `into`, or into the master; only the master is * heard straight out (#325). One that was being let go stays. The context comes first: the one * made now sets up the mix, whose bus this may be, and that one is to be kept (#369). */ private bus(name: string, into: string | null = null): Bus { const context = this.audio(); const known = this.buses.get(name); if (known) { known.leaving = 0; return known; } const target = name === MASTER_MIX ? null : (into ?? MASTER_MIX); const made: Bus = { input: context.createGain(), gain: context.createGain(), pan: context.createStereoPanner(), set: { gain: 1, pan: 0 }, into: target, sends: null, fx: null, chain: null, leaving: 0, meter: null, }; made.input.connect(made.gain).connect(made.pan).connect(this.outOf(target)); this.buses.set(name, made); return made; } private outOf(into: string | null): AudioNode { if (into) return this.bus(into).input; const context = this.audio(); return this.guard?.input ?? context.destination; } /** Sends a bus on into another one where it went somewhere else; the master stays heard. */ private route(name: string, bus: Bus, into: string | null): void { const target = name === MASTER_MIX ? null : (into ?? MASTER_MIX); if (bus.into === target && !bus.sends) return; this.dropSends(bus); this.cutPan(bus); bus.pan.connect(this.outOf(target)); bus.into = target; this.alignSends(); } /** * A track of a song on along each of its sends (#350): from its tap, through a delay that makes it * as late as the latest way of the track, at its level, into the bus it names. Only the levels * glide where the sends are the same ones as before. */ private setSends(bus: Bus, sends: readonly { to: string; gain: number; tap: SendTap }[]): void { const now = this.audio().currentTime; const same = bus.sends?.length === sends.length && bus.sends.every((old, i) => old.to === sends[i]!.to && old.tap === sends[i]!.tap); if (same) { bus.sends!.forEach((old, i) => { old.gain = Math.max(0, sends[i]!.gain); old.level.gain.setTargetAtTime(old.gain, now, GLIDE / 3); }); return; } if (bus.sends) this.dropSends(bus); else this.cutPan(bus); const context = this.audio(); bus.sends = sends.map(({ to, gain, tap }) => { const delay = context.createDelay(MAX_SEND_DELAY); const level = context.createGain(); level.gain.value = Math.max(0, gain); delay.connect(level).connect(this.bus(to).input); return { to, tap, gain: level.gain.value, delay, level }; }); this.wireTaps(bus); this.alignSends(); } /** The node a send takes its sound from: after the pan, after the sound FX, or the input. */ private tapOf(bus: Bus, tap: SendTap): AudioNode { return tap === 'post' ? bus.pan : tap === 'postFx' ? (bus.chain?.out ?? bus.input) : bus.input; } /** Feeds every send from its tap; done again whenever a chain comes or goes, which moves a tap. */ private wireTaps(bus: Bus): void { for (const send of bus.sends ?? []) this.tapOf(bus, send.tap).connect(send.delay); } private dropSends(bus: Bus): void { for (const send of bus.sends ?? []) { try { this.tapOf(bus, send.tap).disconnect(send.delay); } catch { // Already cut where its chain was made again. } send.delay.disconnect(); send.level.disconnect(); } bus.sends = null; } /** Holds every send back as far as `sendDelays` says, so no way of a track is heard before another. */ private alignSends(): void { for (const [name, bus] of this.buses) { if (!bus.sends) continue; const delays = sendDelays(name, this.timed); bus.sends.forEach((send, i) => { send.delay.delayTime.value = Math.min(MAX_SEND_DELAY, delays[i] ?? 0); }); } } /** * The master and the buses of the project (#325): each into the bus it names, at its level unless * a running game set another, through its sound FX. A bus whose asset is gone is let go. Applied * once there is a context; done once the chains are there. The same mix again changes nothing, so * the editor can hand it over on every change of the project. */ async setMix(buses: readonly MixBus[]): Promise { if (sameMix(this.mix, buses)) return; this.mix = buses; if (this.context) await this.applyMix(); } private async applyMix(): Promise { const names = new Set(this.mix.map((bus) => bus.name)); const waits: Promise[] = []; for (const { name, volume, into, sends, fx } of this.mix) { const bus = this.bus(name, into); if (sends) this.setSends(bus, sends); else this.route(name, bus, into); this.glideLevel(bus, this.mixVolumes.get(name) ?? volume); waits.push(this.setFx(bus, fx)); } const gone = [...this.buses.keys()].filter( (name) => name.startsWith(`${MASTER_MIX}:`) && !names.has(name), ); for (const name of gone) this.mixVolumes.delete(name); this.releaseBuses(gone); await Promise.all(waits); } /** A quick glide of the level of a bus to `gain`. */ private glideLevel(bus: Bus, gain: number, time = 0, ease: EaseShare = 'linear'): void { const to = Math.max(0, gain); move(bus.gain.gain, to, this.audio().currentTime, time, false, ease); bus.set = { ...bus.set, gain: to }; } /** * The level a running game sets a bus of the mix to (#325), over `time` seconds along `ease`; it * holds until `resetMix`, whatever the mix is set to meanwhile. */ setMixVolume(name: string, volume: number, time = 0, ease: EaseShare = 'linear'): void { const level = Number.isFinite(volume) ? Math.max(0, Math.min(1, volume)) : 1; this.mixVolumes.set(name, level); if (this.context || this.buses.has(name)) this.glideLevel(this.bus(name), level, Math.max(0, time), ease); } /** The level a bus of the mix is at: what a game set it to, else that of its asset. */ mixVolume(name: string): number { return this.mixVolumes.get(name) ?? this.mix.find((bus) => bus.name === name)?.volume ?? 1; } /** Back to the levels of the assets, as a new run starts and an old one ends. */ resetMix(): void { const set = [...this.mixVolumes.keys()]; this.mixVolumes.clear(); if (!this.context) return; for (const name of set) { const bus = this.buses.get(name); if (bus) this.glideLevel(bus, this.mixVolume(name)); } } /** * The sound FX of a bus of the mix (#325), as `tune` changes those of a voice. Says whether there * is such a bus. */ tuneMix(name: string, params: FxParams, time = 0, ease: EaseShare = 'linear'): boolean { const bus = this.buses.get(name); if (!bus) return this.mix.some((known) => known.name === name); this.tuneBus(bus, params, time, ease); return true; } /** * Sets the sound FX of a bus (#346), its chain made again only where the FX is another or counts by * another tempo. Done once the chain is there, after the worklet it needs. */ private async setFx(bus: Bus, fx: FxVoice | null): Promise { if (bus.fx?.plan === fx?.plan && bus.fx?.tempo === fx?.tempo) return; bus.fx = fx; this.unchain(bus); if (!fx) return; await Promise.all([ needsWorklet(fx.plan) ? this.fxWorklet() : null, ...impulsesOf(fx.plan).map((file) => this.load(file)), ]); if (bus.fx === fx && !bus.chain) this.chain(bus); } /** Puts the chain of its sound FX between the input of a bus and its level. */ private chain(bus: Bus): void { const fx = bus.fx; if (!fx) return; const context = this.audio(); const made: BusChain = { knobs: [], latency: 0, stops: [], nodes: [], out: bus.input }; bus.input.disconnect(); const { out, latency } = this.chainOf(context, fxChainOf(fx.plan, fx.params, fx.tempo), bus.input, made); out.connect(bus.gain); made.latency = latency; made.out = out; bus.chain = made; this.wireTaps(bus); this.alignSends(); } private unchain(bus: Bus): void { const chain = bus.chain; if (!chain) return; bus.chain = null; for (const stop of chain.stops) stop(); for (const node of chain.nodes) node.disconnect(); bus.input.disconnect(); bus.input.connect(bus.gain); this.wireTaps(bus); this.alignSends(); } /** * The nodes a chain of a bus becomes, fed from `input`, in the order `wire` makes those of a voice * so their knobs line up with `levelsOf`. A way has no source of its own: both ways of a Mix are fed * the bus. A Pitch is a shifter of the sound going by (#346), late by a frame, and the other way of * a Mix is held back as far, so the two stay together. Says where it comes out and how late. */ private chainOf( context: BaseAudioContext, chain: FxChain, input: AudioNode, made: BusChain, ): { out: AudioNode; latency: number } { const { knobs, nodes } = made; let tail: AudioNode = input; let latency = 0; if ('mix' in chain) { const both = chain.curve === 'both'; const a = this.chainOf(context, chain.a, input, made); const shareA = gainNode(context, both ? 0 : 1 - chain.mix); if (!both) knobs.push({ param: shareA.gain, target: 1 - chain.mix, even: 'value' }); const b = this.chainOf(context, chain.b, input, made); const shareB = gainNode(context, both ? 0 : chain.mix); if (!both) knobs.push({ param: shareB.gain, target: chain.mix, even: 'value' }); else { const driver = bothAtCenter(context, chain.mix, shareA, shareB); driver.start(); knobs.push({ param: driver.offset, target: chain.mix, even: 'value' }); made.stops.push(() => driver.stop()); nodes.push(driver); } latency = Math.max(a.latency, b.latency); tail = context.createGain(); for (const [way, share] of [ [a, shareA], [b, shareB], ] as const) { let out = way.out; if (way.latency < latency) { const line = context.createDelay(latency); line.delayTime.value = latency - way.latency; out.connect(line); nodes.push(line); out = line; } out.connect(share).connect(tail); nodes.push(share); } nodes.push(tail); } else if (chain.shifted) { const shifted = this.shifter(context, chain.pitch, input, made); tail = shifted.out; latency = shifted.latency; } for (const stage of chain.stages) { // A sound FX has no Envelope; one would only silence the bus. if (stage.kind === 'envelope') continue; const impulse = stage.kind === 'reverb' && stage.impulse ? this.buffer(stage.impulse) : null; const stageMade = stageNodes(context, stage, undefined, context.currentTime, impulse); latency += stageLatency(stage, context.sampleRate); tail.connect(stageMade.input); tail = stageMade.output; knobs.push(...stageMade.knobs); if (stageMade.stop) made.stops.push(stageMade.stop); if (stage.kind === 'tap') made.stops.push(this.spectra.hold(stage, stageMade.output)); nodes.push(stageMade.input, stageMade.output); } return { out: tail, latency }; } /** The Pitch of a way of a bus (#346); without the worklet the sound goes by as it is, on time. */ private shifter( context: BaseAudioContext, pitch: number, input: AudioNode, made: BusChain, ): { out: AudioNode; latency: number } { let node: AudioWorkletNode; try { if (!this.hasWorklet) throw new Error('no worklet'); node = new AudioWorkletNode(context, 'pitch-shifter', { numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [2], channelCount: 2, channelCountMode: 'explicit', parameterData: { pitch }, }); } catch { made.knobs.push({ param: null, target: pitch, even: 'value' }); return { out: input, latency: 0 }; } input.connect(node); made.nodes.push(node); made.stops.push(() => stopProcessor(node)); made.knobs.push({ param: node.parameters.get('pitch')!, target: pitch, even: 'value' }); return { out: node, latency: shifterLatency(context.sampleRate) }; } /** * Seconds after its sound stops that a bus is still heard (#346): the echoes of its chain, after * what it and the buses it goes into hold back. */ private ringOf(name: string): number { const fx = this.buses.get(name)?.fx; const tail = fx && fxRings(fx.plan) ? fxTailOf(fxChainOf(fx.plan, fx.params, fx.tempo), this.soundLength) : 0; return tail + this.busLatency(name); } private dropBus(name: string, bus: Bus): void { this.dropMeter(bus); this.dropSends(bus); this.unchain(bus); bus.input.disconnect(); bus.gain.disconnect(); bus.pan.disconnect(); this.buses.delete(name); } /** * A sequence without notes or clips plays nothing, and is given no number. Each playing is heard * through a master of its own, silent until it starts (#336): its level and its fades. The tracks * of a song go into it, each set as it was given; a note or clip naming no track goes straight in. */ playSequence(name: string, play: SequencePlay, owner = ''): number { if (play.sequence.notes.length === 0 && !play.sequence.clips?.length) return 0; const id = this.nextId++; // A bake whose file could not be had is no bake: the notes are played instead (#340). const { bake: asked, ...rest } = play; const bake = asked && !this.hasFailed(asked.file) ? asked : null; const buses = bake ? [] : (play.buses ?? []); this.open(id, buses, play.fx ?? null, play.bus ?? MASTER_MIX, play.sends); const tracks = new Set(buses.map((bus) => bus.name)); const into = (bus: string | undefined) => groupBus(id, bus !== undefined && tracks.has(bus) ? bus : ''); const { strike, strikeClip } = play; const chains = [...buses.flatMap((bus) => bus.fx ?? []), ...(play.fx ? [play.fx] : [])]; const needs = bake ? { layers: [], files: [bake.file] } : play.needs; const routed: SequencePlay = { ...rest, ...(bake && { bake }), needs: { ...needs, layers: [...needs.layers, ...chains.map((fx) => fx.plan)] }, strike: (note) => strike({ ...note, bus: into(note.bus) }), ...(strikeClip && { strikeClip: (clip: ScheduledClip) => strikeClip({ ...clip, bus: into(clip.bus) }), }), }; this.sequences.start(id, name, routed, owner); return id; } stopSequence(name: string, letRing = false, fade = 0): void { this.sequences.stopNamed(name, letRing, fade); } stopPlayback(id: number, fade: number): void { if (this.sequences.has(id)) this.sequences.stopId(id, false, fade); else this.stopVoice(id); } /** * The patterns and songs playing, oldest first, with the notes and clips last planned of each, and * whether it plays its bake (test hook). */ playbacks(): { id: number; name: string; owner: string; playing: boolean; fading: number | null; planned: ScheduledNote[]; clips: ScheduledClip[]; baked: boolean; startedAt: number | null; }[] { return this.sequences.current().map(({ id, name, owner, runner, fading, planned, clips, baked }) => ({ id, name, owner, playing: runner.playing, startedAt: runner.startedAt, fading, planned: [...planned], clips: [...clips], baked, })); } /** * What goes into the master of a playing, taken off before its level into `to` (#340): what a bake * keeps. Says whether there was such a playing. */ tapPlaying(id: number, to: AudioNode): boolean { const master = this.buses.get(groupBus(id, '')); if (!master) return false; master.input.disconnect(); master.input.connect(to); return true; } /** * The bake of a playing (#340) as one voice on its master, heard from `heard`: its file once to its * end, faded out there, or up to the loop end and then round its second round. Streamed where it can * be (#349), else decoded – in one source where the round follows at once, else in a second that * takes over to the sample. 0 without its bytes. */ private playBaked(id: number, bake: BakePlay, heard: number): number { const stream = this.hasWorklet ? this.streamed(bake.file) : null; const streaming = stream?.head ? stream : null; const buffer = streaming ? null : this.buffer(bake.file); if (!streaming && !buffer) return 0; const context = this.audio(); const bus = groupBus(id, ''); const at = Math.max(context.currentTime, heard - this.busLatency(bus)); const gain = context.createGain(); gain.gain.value = Math.max(0, Math.min(MAX_VOLUME, bake.gain)); gain.connect(this.busInput(bus)); const { end } = bake.playback; const played = streaming ? this.bakeStreamed(streaming, bake.playback, gain, at) : this.bakeDecoded(buffer!, bake.playback, gain, at); const voice: Voice = { id: this.nextId++, file: bake.file, owner: '', bus, at, sources: played.sources, ending: new Set(), drivers: [], envelopes: [], finish: null, gain, fx: { layers: [], params: {}, tempo: DEFAULT_TEMPO }, knobs: [], rings: false, ringing: false, quiet: null, vocoders: 0, released: false, ringFade: BAKE_LET_RING, }; const going = new Set(voice.sources); voice.finish = (source) => { if (going.delete(source) && going.size === 0) this.ended(voice); }; for (const source of played.sources) source.ended(() => voice.finish?.(source)); this.voices.push(voice); this.started += 1; this.last = { file: bake.file, volume: gain.gain.value, loop: bake.playback.round !== null, layers: 0, pitches: [], stages: 0, duration: played.duration, at, envelopes: 0, bus, }; if (end !== null) { gain.gain.setValueAtTime(gain.gain.value, at + Math.max(0, end - FADE)); gain.gain.linearRampToValueAtTime(0, at + end); } played.start(); return voice.id; } /** A bake streamed (#349): one `stream-player` along the same plan, the round looped to the sample. */ private bakeStreamed( stream: StreamFile, { end, until, round }: BakePlay['playback'], gain: GainNode, at: number, ): { sources: Source[]; start: () => void; duration: number } { const frames = (seconds: number) => Math.round(seconds * STREAM_RATE); const plan: StreamPlan = end !== null ? { from: 0, until: frames(end), loop: null } : { from: 0, until: frames(until), loop: round && { start: frames(round.start), end: frames(round.end) }, }; const { node, source } = this.streamNode(this.audio(), stream, plan, at); node.connect(gain); return { sources: [source], start: () => {}, duration: stream.frames / STREAM_RATE }; } /** A bake decoded, in one buffer source or two. */ private bakeDecoded( buffer: AudioBuffer, { end, until, round }: BakePlay['playback'], gain: GainNode, at: number, ): { sources: Source[]; start: () => void; duration: number } { const context = this.audio(); const made = (from: number, loop: { start: number; end: number } | null) => { const node = context.createBufferSource(); node.buffer = buffer; if (loop) { node.loop = true; node.loopStart = loop.start; node.loopEnd = loop.end; } node.connect(gain); const source: Source = { start: () => {}, stop: () => node.stop(), ended: (then) => { node.onended = then; }, fade: null, gate: false, leave: null, }; return { node, source, from }; }; const parts = round && round.start === until ? [made(0, round)] : round ? [made(0, null), made(round.start, round)] : [made(0, null)]; const [head, next] = parts; const start = () => { if (end !== null) head!.node.start(at, 0, end); else if (next) { head!.node.start(at, 0, until); next.node.start(at + until, next.from); } else head!.node.start(at, 0); }; return { sources: parts.map((part) => part.source), start, duration: buffer.duration }; } /** * The buses of a playing, its tracks into its master and a pattern item's into its track (#346), * each with the sound FX it was given; its master into `into`, or along `sends` (#351). Their * chains are made once the worklet is there (`ready`). */ private open( id: number, tracks: readonly BusLevel[], fx: FxVoice | null, into: string, sends?: readonly MixSend[], ): void { const masterName = groupBus(id, ''); const master = this.bus(masterName, into); master.gain.gain.value = 0; master.set = { gain: 0, pan: 0 }; master.fx = fx; if (sends) this.setSends(master, sends); const names = new Set(tracks.map((track) => track.name)); const inside = (bus: string | undefined) => bus !== undefined && names.has(bus) ? groupBus(id, bus) : masterName; const made = tracks.map(({ name, gain, pan, fx: trackFx }) => { const track = this.bus(groupBus(id, name), masterName); track.gain.gain.value = Math.max(0, gain); track.pan.pan.value = Math.max(-1, Math.min(1, pan)); track.set = { gain: track.gain.gain.value, pan: track.pan.pan.value }; track.fx = trackFx ?? null; return track; }); tracks.forEach(({ name, into, sends }, i) => { const track = made[i]!; if (sends) this.setSends( track, sends.map((send) => ({ ...send, to: inside(send.to) })), ); else this.route(groupBus(id, name), track, inside(into)); }); this.groups.add(id); } /** What a playing needs loaded, then the chains of its buses (#346): before its first note is planned. */ private async ready(needs: SequenceNeeds, id: number): Promise { await this.prepare(needs.layers, needs.files); for (const name of this.groupBuses(id)) { const bus = this.buses.get(name); if (bus?.fx && !bus.chain) this.chain(bus); } } /** The names of the buses of a playing: its master and every track inside it. */ private groupBuses(id: number): string[] { const master = groupBus(id, ''); return [...this.buses.keys()].filter((bus) => bus === master || bus.startsWith(`${master}:`)); } /** How late the latest way through the buses of a playing is (#346): how early its notes start. */ private groupAhead(id: number): number { return Math.max(0, ...this.groupBuses(id).map((name) => this.busLatency(name))); } /** The master of a playing, from where it stands at `at` to `gain`, there `seconds` later. */ private level(id: number, gain: number, at: number, seconds: number): void { const master = this.buses.get(groupBus(id, '')); if (!master) return; const param = master.gain.gain; const from = param.value; param.cancelScheduledValues(at); param.setValueAtTime(seconds > 0 ? from : gain, at); if (seconds > 0) param.linearRampToValueAtTime(gain, at + seconds); master.set = { ...master.set, gain }; } /** * Lets go of the buses of every playing that is gone and has nothing left on them – a fade later, * so a voice cut a moment ago is not cut off mid-fade. */ private sweep(): void { if (this.groups.size === 0) return; for (const id of [...this.groups]) { if (!this.groups.has(id) || this.sequences.has(id)) continue; const on = (bus: string) => bus === groupBus(id, '') || bus.startsWith(`${groupBus(id, '')}:`); if (this.voices.some((voice) => on(voice.bus))) continue; if (this.pending.some((waiting) => !waiting.stopped && on(waiting.bus))) continue; this.groups.delete(id); const names = [...this.buses.keys()].filter(on); // Its echoes, and what its shifters still hold, are heard after its last voice (#346). const ring = Math.max(0, ...names.map((name) => this.ringOf(name))); this.after(this.audio().currentTime + FADE + ring, () => { for (const name of names) { const bus = this.buses.get(name); if (bus) this.dropBus(name, bus); } }); } } /** * The end of a note (#331): the ways that end with it fade out, a looping one that does not leaves * its loop, the others play on, and the voice goes once they have all ended – or rings on first * where it has echoes. `at` is the second of the audio clock it ends at (#332): the fade is set on * the clock to the sample, leaving a loop waits for the page. */ release(id: number, at = 0): void { const waiting = this.pending.find((other) => other.id === id); if (waiting) waiting.released = true; const voice = this.voices.find((other) => other.id === id); if (!voice) return; this.letGo(voice, Math.max(at, this.audio().currentTime), false); } /** * Ends a note at `when`, or with `all` lets a stopped one ring (#335). Its Envelopes fall away in * their release, and the sources whose volume they are end with it – but one under only one-shots * holds, unless `all`. Of the rest the gated ones fade out at once, with `all` every one. */ private letGo(voice: Voice, when: number, all: boolean): void { voice.released = true; for (const run of voice.envelopes) { if (run.released) continue; if (!all && run.sources.length > 0 && run.sources.every(playsThrough)) continue; run.released = true; releaseEnvelope(run.driver.offset, run.envelope, voice.at, when); this.endAt(voice, run.sources, when + releaseTime(run.envelope)); } const context = this.audio(); const leaving = voice.sources.filter((source) => !source.gate && source.leave); const leave = () => { for (const source of leaving) source.leave!(); }; if (leaving.length > 0 && when > context.currentTime) this.after(when, () => this.voices.includes(voice) && leave()); else leave(); this.endAt( voice, voice.sources.filter((source) => all || source.gate), when, ); } /** Fades these sources of a voice out from `at` and lets them go then, each only once. */ private endAt(voice: Voice, sources: readonly Source[], at: number): void { const going = sources.filter((source) => !voice.ending.has(source)); if (going.length === 0) return; for (const source of going) { voice.ending.add(source); if (source.fade) fadeOut(source.fade.gain, at); } this.after(at + FADE, () => { if (!this.voices.includes(voice)) return; for (const source of going) { source.stop(); voice.finish?.(source); } }); } private begin(start: Start): number { const id = this.nextId++; const asked: Start = { ...start, id }; // A Reverb's sound (#328) is waited for as well, but a voice is not silent for lack of it. const impulseFiles = start.fx.layers.flatMap(impulsesOf); // A streamed sound (#349) waits for the head of its copy and the worklet that plays it instead. const { stream } = start; const loaded = [...(stream ? [] : start.files), ...impulseFiles].every((file) => this.buffer(file) !== null) && (!stream || stream.head !== null); const worklet = stream || start.fx.layers.some(needsWorklet) ? this.fxWorklet() : null; if (loaded && (!worklet || this.hasWorklet !== null)) { this.start(asked); return id; } const askedAt = this.cancelsFor(start.file, start.owner); const waiting: Pending = { id, bus: start.bus ?? '', fx: { ...start.fx, params: { ...start.fx.params } }, timed: [], stopped: false, released: false, }; this.pending.push(waiting); const impulsesLoaded = Promise.all(impulseFiles.map((file) => this.load(file))); // A copy that cannot be streamed leaves the sound decoded, as it would be without one. const sounds = stream ? stream.ready().then(async (ready) => [ready || (await this.load(start.file)) !== null]) : Promise.all(start.files.map((file) => this.load(file))); void Promise.all([sounds, worklet, impulsesLoaded]).then(([buffers]) => { this.pending = this.pending.filter((other) => other !== waiting); // A `Stop sound` – or the destruction of the behavior that asked – that happened while the // bytes were in flight has to win, or a sound the run already ended would start afterwards. const silent = buffers.every((buffer) => !buffer) && !start.fx.layers.some(hasChip); if (silent || waiting.stopped) return; if (this.cancelsFor(start.file, start.owner) !== askedAt) return; const started: Start = { ...asked, fx: waiting.fx }; if (stream?.failed) delete started.stream; this.start(started); for (const { params, time, ease } of waiting.timed) this.tune(id, params, time, ease); if (waiting.released) this.release(id); }); return id; } /** Says whether the voice was still there to be reached – playing, or waiting on its bytes. */ tune(id: number, params: FxParams, time = 0, ease: EaseShare = 'linear'): boolean { const waiting = this.pending.find((other) => other.id === id); if (waiting) { // A quick change is where the voice starts; a timed one – and anything after it, so the order // holds – runs from the moment it is heard. if (time > 0 || waiting.timed.length > 0) waiting.timed.push({ params, time, ease }); else waiting.fx = { ...waiting.fx, params: { ...waiting.fx.params, ...params } }; return true; } const voice = this.voices.find((other) => other.id === id); if (!voice) return this.sequences.has(id) && this.tuneMaster(id, params, time, ease); voice.fx = { ...voice.fx, params: { ...voice.fx.params, ...params } }; if (this.glide(voice, time, ease)) this.tunedAt = this.audio().currentTime; // A longer feedback makes for a longer tail: it is counted again from where the change ends. if (voice.ringing) this.ringFor(voice, Math.max(0, time) + this.tailOf(voice.fx)); this.tuned += 1; return true; } /** * The sound FX of a pattern or song playing (#346), on its master; one still loading starts with * the values. Says it was reached, whether or not it has a sound FX. */ private tuneMaster(id: number, params: FxParams, time: number, ease: EaseShare): boolean { const master = this.buses.get(groupBus(id, '')); if (master) this.tuneBus(master, params, time, ease); return true; } private tuneBus(bus: Bus, params: FxParams, time: number, ease: EaseShare): void { const fx = bus.fx; if (!fx) return; bus.fx = { ...fx, params: { ...fx.params, ...params } }; // The chain stays: `setFx` would see the same plan, and the knobs move it. const levels = levelsOf(fxChainOf(fx.plan, bus.fx.params, fx.tempo)); if (bus.chain && glideKnobs(bus.chain.knobs, levels, this.audio().currentTime, time, ease)) this.tunedAt = this.audio().currentTime; this.tuned += 1; } /** * Seconds a voice rings on after its sound has ended, at the values it has now (#324); 0 for a * voice without echoes and for one that is not there. */ tail(id: number): number { const voice = this.voices.find((other) => other.id === id) ?? this.pending.find((other) => other.id === id); return voice ? this.tailOf(voice.fx) : 0; } /** The second of the audio clock a voice started at; null while it waits on its bytes or is gone. */ startedAt(id: number): number | null { return this.voices.find((voice) => voice.id === id)?.at ?? null; } /** Whether a voice is still heard – or waiting on its bytes to be. A pattern or song counts as one. */ playing(id: number): boolean { return this.sounding(id) || this.sequences.has(id); } private sounding(id: number): boolean { return this.voices.some((voice) => voice.id === id) || this.pending.some((other) => other.id === id); } /** `letRing` stops the sound itself and lets its echoes fade as they would (#324). */ stop(file: string, letRing = false): void { if (file === '') this.sequences.stopNamed('', letRing); this.silence(file === '' ? this.voices : this.voices.filter((voice) => voice.file === file), letRing); if (file === '') this.allStops += 1; else this.stops.set(file, (this.stops.get(file) ?? 0) + 1); } /** One voice by its number, also one still waiting on its bytes, or a pattern. */ stopVoice(id: number, letRing = false): void { this.sequences.stopId(id, letRing); const waiting = this.pending.find((other) => other.id === id); if (waiting) waiting.stopped = true; this.silence( this.voices.filter((voice) => voice.id === id), letRing, ); } /** * Every voice of one behavior. An empty owner is deliberately no one: those are the voices that * were started to outlive their behavior, so "stop the owner" must not reach them. */ stopOwner(owner: string): void { if (owner === '') return; this.sequences.stopOwner(owner); this.silence(this.voices.filter((voice) => voice.owner === owner)); this.ownerStops.set(owner, (this.ownerStops.get(owner) ?? 0) + 1); } /** Every voice at once: what ending a run does, so nothing outlives the world that started it. */ stopAll(): void { this.stop(''); } /** * A pause holds the sounds where they are; resuming carries them on mid-note. Until then nothing * wakes the clock – a pattern or song reading it every few milliseconds would undo the pause. */ setSuspended(suspended: boolean): void { this.held = suspended; const context = this.live(); if (!context) return; if (suspended) void context.suspend(); else void context.resume(); } /** What has been played and what is playing – the only witness a muted check script has. */ stats(): { playing: number; started: number; stopped: number; tuned: number; tunedAt: number | null; decoded: number; /** Bytes of samples in the files decoded whole: what a long sound costs when it does not stream. */ decodedBytes: number; failed: number; /** Whether the pitch worklet is there; null while nothing has asked for it or it is loading. */ worklet: boolean | null; /** Ways playing through the pitch worklet now, at most `MAX_VOCODERS`. */ vocoders: number; last: LastVoice | null; /** * Streaming (#349): sources playing from a copy now, samples that came too late to be heard, the * bytes fetched of every copy, the copies whose head is decoded, and the bytes of samples held for * streaming: every head and what is sent ahead to the worklet. */ streaming: { playing: number; skipped: number; fetched: number; heads: number; held: number }; } { const files = [...this.streamFiles.values()]; return { playing: this.voices.length, started: this.started, stopped: this.stopped, tuned: this.tuned, tunedAt: this.tunedAt, decoded: this.buffers.size, decodedBytes: [...this.buffers.values()].reduce( (sum, buffer) => sum + buffer.length * buffer.numberOfChannels * Float32Array.BYTES_PER_ELEMENT, 0, ), failed: this.failed.size, worklet: this.hasWorklet, vocoders: this.voices.reduce((sum, voice) => sum + vocodersOf(voice), 0), last: this.last, streaming: { playing: this.feeds.size, skipped: [...this.feeds].reduce((sum, feed) => sum + feed.skipped, this.skippedBefore), fetched: files.reduce((sum, file) => sum + file.bytes, 0), heads: files.filter((file) => file.head !== null).length, held: files.reduce((sum, file) => sum + file.headBytes, 0) + [...this.feeds].reduce((sum, feed) => sum + feed.ahead, 0), }, }; } /** * Where the knobs of each voice of that file are right now, oldest first, a pitch in semitones – * what a check reads to see a change arrive at one voice and not another (test hook). */ levels(file: string): number[][] { return this.voices .filter((voice) => voice.file === file) .map((voice) => voice.knobs.map((knob) => knob.param?.value ?? knob.target)); } /** * Seconds on the audio clock, 0 before there is one (test hook): what a check times a voice by, * since that clock and the page's need not run alike. */ clock(): number { return this.context?.currentTime ?? 0; } /** `running`, `suspended` by a pause, or `none` before there is a context (test hook). */ clockState(): string { return this.context?.state ?? 'none'; } /** Seconds on the audio clock, opening the context where there is none yet: what the scheduler plans by (#332). */ time(): number { const context = this.audio(); if (context.state === 'suspended' && !this.held) void this.live()?.resume(); return context.currentTime; } /** The context everything plays on, opened where there is none yet: a take is recorded on its clock (#342). */ liveContext(): AudioContext { this.audio(); return this.live()!; } /** * The sound of a video (#402) into the master, as a voice goes: from now on `element` is heard * through the mix and only through it. The gain is its volume. Null while a bake has the context. */ mediaInput(element: HTMLMediaElement): GainNode | null { this.audio(); const context = this.live(); if (!context) return null; if (context.state === 'suspended' && !this.held) void context.resume(); const gain = context.createGain(); context.createMediaElementSource(element).connect(gain).connect(this.busInput('')); return gain; } /** The context it plays to the speakers on; null before there is one, and for a bake (#340). */ private live(): AudioContext | null { return this.render ? null : (this.context as AudioContext | null); } /** * Seconds between the audio clock and what is heard: the limiter at the output (#359) and the * device, where the browser says. */ latency(): number { const context = this.live(); if (!context) return 0; const guard = this.guard?.latency ?? 0; // Older Safari and Firefox have no `outputLatency`, some no `baseLatency` either. const out = context.outputLatency; if (Number.isFinite(out) && out > 0) return guard + out; return guard + (Number.isFinite(context.baseLatency) ? context.baseLatency : 0); } /** * Loads every file and the worklet these notes need, so the first note the scheduler plans is * not late waiting on its bytes (#332). */ async prepare(layers: readonly FxPlan[], files: readonly string[] = []): Promise { // A Sample or a Reverb's sound is always decoded; a sound or a bake may stream (#349). const decoded = new Set([...layers.flatMap(filesOf), ...layers.flatMap(impulsesOf)]); const streamed = files.flatMap((file) => { const stream = decoded.has(file) ? null : this.streamed(file); return stream ? [{ file, stream }] : []; }); const loaded = [...new Set([...decoded, ...files])].filter( (file) => file !== '' && !streamed.some((entry) => entry.file === file), ); await Promise.all([ ...loaded.map((file) => this.load(file)), ...streamed.map(async ({ file, stream }) => { if (!(await stream.ready())) await this.load(file); }), layers.some(needsWorklet) || streamed.length > 0 ? this.fxWorklet() : null, ]); } /** Loads the FX worklet into its context, for a node of it made outside (#340); says whether it is there. */ worklet(): Promise { return this.fxWorklet(); } /** Where the FX worklet is served from, for a check that runs it in an offline context (test hook). */ fxWorkletUrl(): string { return fxWorkletUrl; } /** * How long a decoded sound runs, in seconds – a cut (#343) as long as what it keeps; null while it * has not been loaded (test hook). */ duration(file: string): number | null { const range = this.rangeOf(file); if (range) return (range.to - range.from) / range.whole.sampleRate; // A streamed one (#349) is as long as its copy, known without decoding a sample of it. const stream = this.streams.get(file); return stream ? stream.frames / STREAM_RATE : null; } /** The decoded samples of a file as it is, without a cut; null while it has not been loaded. */ decoded(file: string): AudioBuffer | null { return this.buffers.get(file) ?? null; } /** Whether a file could not be fetched or decoded – it is not tried again. */ failedToLoad(file: string): boolean { return this.hasFailed(file); } length(file: string): number { return this.duration(file) ?? 0; } /** * The loudest sample of each of `PEAK_BUCKETS` equal stretches of a loaded sound, from 0 to 1 – * what a clip of a song draws as its waveform (#334); null while it has not been loaded. */ peaks(file: string): Float32Array | null { const known = this.peaked.get(file); if (known) return known; const range = this.rangeOf(file); if (!range) return null; const peaks = peaksOf(range.whole, range); this.peaked.set(file, peaks); return peaks; } /** * Loads what `peaks` and `duration` read for a name. A streamed one (#349) is not loaded whole for * it: its copy is decoded at `PICTURE_RATE`, its peaks kept and the samples let go. */ loadPicture(name: string): Promise { const stream = this.streamed(name); if (!stream) return this.load(name).then(() => {}); if (this.peaked.has(name)) return Promise.resolve(); let picture = this.pictures.get(name); if (!picture) { picture = this.draw(name, stream); this.pictures.set(name, picture); } return picture; } private async draw(name: string, stream: StreamFile): Promise { try { const bytes = await stream.whole(); const picture = await new OfflineAudioContext(1, 1, PICTURE_RATE).decodeAudioData(bytes); this.peaked.set(name, peaksOf(picture, { from: 0, to: picture.length, reverse: false })); } catch (error: unknown) { console.error(`[sounds] cannot draw ${name}`, error); } } /** * Starts loading every file not held yet. Waiting for the first play left that play late: its * start was not heard, and a graph tuning it in the meantime had moved on by the time it started. * One that streams (#349) has the head of its copy loaded instead. */ preload(files: Iterable): void { for (const file of files) { if (!file) continue; const stream = this.streamed(file); if (stream) void stream.ready(); else void this.load(file); } } /** * Fetches and decodes once per file; a second play while it is in flight waits on the first. A cut * (#343) loads its file and is made from it – the cut of a clip with a copy (#349) loads that * instead, unless the whole file is there already. */ async load(file: string): Promise { if (parseCutName(file).cut) { const copy = this.copyOf(file); if (copy && !this.buffers.has(parseCutName(file).file) && (await this.load(copy))) return this.buffer(file); const whole = await this.load(parseCutName(file).file); return whole ? this.buffer(file) : null; } const held = this.buffers.get(file); if (held) return held; if (this.failed.has(file)) return null; const running = this.loading.get(file); if (running) return running; const work = (async () => { try { const buffer = this.render ? await this.render.files.load(file) : await this.fetched(file); if (!buffer) throw new Error('not there where it is baked from'); this.buffers.set(file, buffer); return buffer; } catch (error: unknown) { this.failed.add(file); console.error(`[sounds] cannot load ${file}`, error); return null; } finally { this.loading.delete(file); } })(); this.loading.set(file, work); return work; } /** A file decoded at `rate` and not kept: what a stream copy is made from (#349). */ async decodeApart(file: string, rate: number): Promise { const response = await fetch(`${this.base}/${file}`); if (!response.ok) throw new Error(`HTTP ${response.status}`); return new OfflineAudioContext(1, 1, rate).decodeAudioData(await response.arrayBuffer()); } private async fetched(file: string): Promise { const response = await fetch(`${this.base}/${file}`); if (!response.ok) throw new Error(`HTTP ${response.status}`); return this.decoder(file).decodeAudioData(await response.arrayBuffer()); } /** * Ends these voices now: `onended` only fires next frame and the counts have to be right here. A * voice is faded out rather than cut, so it does not click. With `letRing` one with echoes only * loses its sound and stays until they have faded (#324); one already ringing is left to it. A * note with Envelopes falls away in their release first (#335). */ private silence(going: Voice[], letRing = false): void { this.stopped += going.filter((voice) => !(letRing && voice.ringing)).length; for (const voice of going) { if (letRing && voice.ringFade > 0) { this.cut(voice, voice.ringFade); continue; } if (letRing && voice.envelopes.length > 0) { this.letGo(voice, this.audio().currentTime, true); continue; } if (letRing && voice.rings) { if (!voice.ringing) this.ringOut(voice); continue; } this.cut(voice); } } /** * Cuts the oldest pitched voices, those let go first, while more ways than `MAX_VOCODERS` would run * a phase vocoder: a quick run up the keys of a sampled instrument started more than the audio * thread keeps up with, and it fell silent until they had played out. */ private makeRoom(newest: Voice): void { let count = this.voices.reduce((sum, voice) => sum + vocodersOf(voice), 0); if (count <= MAX_VOCODERS) return; const older = this.voices.filter((voice) => voice !== newest && vocodersOf(voice) > 0); for (const voice of [...older.filter((v) => v.released), ...older.filter((v) => !v.released)]) { if (count <= MAX_VOCODERS) return; count -= vocodersOf(voice); this.cut(voice); } } /** Fades the whole voice out over `seconds` and lets it go. */ private cut(voice: Voice, seconds = FADE): void { this.forget(voice); const context = this.audio(); const done = () => { for (const source of voice.sources) source.stop(); for (const driver of voice.drivers) driver.stop(); voice.gain.disconnect(); }; // A suspended clock would hold the fade – and the sound – until the next run resumes it. An // offline one is suspended whenever the page gets to plan, and runs on after. if (context.state !== 'running' && !this.render) return done(); fadeOut(voice.gain.gain, context.currentTime, seconds); this.after(context.currentTime + seconds, done); } /** Fades the sources of a voice with echoes out and lets the echoes ring on (#324). */ private ringOut(voice: Voice): void { voice.ringing = true; const context = this.audio(); for (const source of voice.sources) if (source.fade) fadeOut(source.fade.gain, context.currentTime); this.after(context.currentTime + FADE, () => { for (const source of voice.sources) source.stop(); }); this.ringFor(voice, FADE + this.tailOf(voice.fx)); } /** The sources of a voice have ended by themselves: it goes, or rings on first where it has echoes. */ private ended(voice: Voice): void { if (voice.ringing || !this.voices.includes(voice)) return; const tail = voice.rings ? this.tailOf(voice.fx) : 0; if (tail <= 0) return this.done(voice); voice.ringing = true; this.ringFor(voice, tail); } /** (Re)sets the clock that ends a ringing voice, `seconds` from now. */ private ringFor(voice: Voice, seconds: number): void { voice.quiet?.(); voice.quiet = this.after(this.audio().currentTime + seconds, () => { voice.quiet = null; this.done(voice); }); } /** A voice whose sound and echoes have run out: out of the count, and its nodes let go. */ private done(voice: Voice): void { this.forget(voice); for (const driver of voice.drivers) driver.stop(); // Left on its bus, every node of it would be worked on in every block until it happened to be // collected – an echo, a loop of nodes, for good. What its stages hold back is heard first. const late = voice.rings ? 0 : this.lateOf(voice.fx); if (late > 0) this.after(this.audio().currentTime + late + FADE, () => voice.gain.disconnect()); else voice.gain.disconnect(); } /** Seconds the latest layer of a voice is late by its own stages (#326). */ private lateOf(fx: Layered): number { const rate = this.audio().sampleRate; return Math.max(0, ...fx.layers.map((plan) => fxLatencyOf(fxChainOf(plan, fx.params, fx.tempo), rate))); } private forget(voice: Voice): void { voice.quiet?.(); voice.quiet = null; this.voices = this.voices.filter((other) => other !== voice); if (voice.bus !== '') this.sweep(); } /** * The longest tail of its layers: they are heard side by side. What its stages hold back (#326) * is still to be heard once its sources end, too. */ private tailOf(fx: Layered): number { const rate = this.audio().sampleRate; return fx.layers.reduce((longest, plan) => { if (!fxRings(plan)) return longest; const chain = fxChainOf(plan, fx.params, fx.tempo); return Math.max(longest, fxTailOf(chain, this.soundLength) + fxLatencyOf(chain, rate)); }, 0); } /** * Calls `then` once the audio clock reaches `at` – the clock, not the page's, so a paused game * holds a tail where it is. A silent source that ends then is the only timer Web Audio has. Gives * back what calls it off. */ private after(at: number, then: () => void): () => void { const context = this.audio(); const clock = context.createConstantSource(); clock.offset.value = 0; clock.connect(context.destination); clock.onended = () => { clock.disconnect(); then(); }; clock.start(); clock.stop(at); return () => { clock.onended = null; clock.disconnect(); }; } private start({ id, file, files, volume, loop, owner, fx, duration, at: heard, clip: asked, bus = '', stream, }: Start): void { const context = this.audio(); // A context starts suspended where the page has not been interacted with; without this the // voice would run silently and end as if it had played. if (context.state === 'suspended' && !this.held) void this.live()?.resume(); // As much earlier as its bus and its own stages are late, so it is heard when it was planned // (#346, #326). A layer that is less late is held back to the latest. const chains = fx.layers.map((plan) => fxChainOf(plan, fx.params, fx.tempo)); const lates = chains.map((chain) => fxLatencyOf(chain, context.sampleRate)); const late = Math.max(0, ...lates); const early = this.busLatency(bus) + late; const at = heard > 0 ? heard - early : heard; const clip = asked && { ...asked, end: asked.end - early }; const when = Math.max(at, context.currentTime); // A clip whose bytes came too late is over already, or starts as far in as it would be by now. if (clip && clip.end <= when) return; const gain = context.createGain(); gain.gain.value = Math.max(0, Math.min(MAX_VOLUME, volume)); const rings = fx.layers.some(fxRings); const wiring: Wiring = { context, at: when, // Held here, not looked up later: a cut can be let go while its voice plays. buffers: new Map( [...files, ...fx.layers.flatMap(impulsesOf)].flatMap((name) => { const buffer = this.buffer(name); return buffer ? [[name, buffer] as const] : []; }), ), file, loop, rings, offset: clip ? clip.offset + (when - at) : 0, clip: clip !== undefined, stream: stream ?? null, sources: [], drivers: [], envelopes: new Map(), pitches: [], knobs: [], stages: 0, vocoders: 0, }; chains.forEach((chain, i) => heldBack(context, this.wire(wiring, chain), late - lates[i]!).connect(gain)); gain.connect(this.busInput(bus)); const { sources } = wiring; this.last = { file, volume: gain.gain.value, loop, layers: fx.layers.length, pitches: wiring.pitches, stages: wiring.stages, duration: Math.max(0, ...files.map((name) => this.duration(name) ?? 0)), at: when, envelopes: wiring.envelopes.size, bus, }; const voice: Voice = { id, file, owner, bus, at: when, sources, ending: new Set(), drivers: wiring.drivers, envelopes: [...wiring.envelopes.values()], finish: null, gain, fx, knobs: wiring.knobs, rings, ringing: false, quiet: null, vocoders: wiring.vocoders, released: false, ringFade: 0, }; const going = new Set(sources); voice.finish = (source) => { if (going.delete(source) && going.size === 0) this.ended(voice); }; for (const source of sources) source.ended(() => voice.finish?.(source)); this.voices.push(voice); this.started += 1; this.makeRoom(voice); for (const driver of wiring.drivers) driver.start(); for (const source of sources) source.start(when, wiring.offset); // Silent once its decay is over, however long the note is held (#329). for (const run of voice.envelopes) if (run.envelope.sustain <= 0) this.endAt(voice, run.sources, when + run.envelope.attack + run.envelope.decay); if (duration > 0) this.release(id, when + duration); if (clip) this.endAt(voice, sources, clip.end); } /** * The nodes a chain becomes, up to the one that carries its sound on. Every way has a source of * its own, so each can have its own pitch; a Mix weighs each of its sounds by its share before * they are summed. */ private wire(wiring: Wiring, chain: FxChain): AudioNode { const { context, knobs } = wiring; const follow = this.follower(wiring); const first = wiring.sources.length; let tail: AudioNode; if ('mix' in chain) { tail = context.createGain(); const both = chain.curve === 'both'; // An Envelope moving the share stands in for the knob: a parameter no longer reaches it. const moved = chain.mixMod; // The later way sets when both are heard; the other is held back to it (#326). const lateA = fxLatencyOf(chain.a, context.sampleRate); const lateB = fxLatencyOf(chain.b, context.sampleRate); const wayA = heldBack(context, this.wire(wiring, chain.a), Math.max(lateA, lateB) - lateA); const shareA = gainNode(context, both || moved ? 0 : 1 - chain.mix); wayA.connect(shareA).connect(tail); if (!both) { knobs.push({ param: moved ? null : shareA.gain, target: 1 - chain.mix, even: 'value' }); if (moved) follow(moved, shareA.gain, false, (mix) => 1 - mix); } const wayB = heldBack(context, this.wire(wiring, chain.b), Math.max(lateA, lateB) - lateB); const shareB = gainNode(context, both || moved ? 0 : chain.mix); wayB.connect(shareB).connect(tail); if (!both) { knobs.push({ param: moved ? null : shareB.gain, target: chain.mix, even: 'value' }); if (moved) follow(moved, shareB.gain, false); } else { const driver = bothAtCenter(context, moved ? 0 : chain.mix, shareA, shareB); knobs.push({ param: moved ? null : driver.offset, target: chain.mix, even: 'value' }); if (moved) follow(moved, driver.offset, false); wiring.drivers.push(driver); } } else tail = this.source(wiring, chain); const under = wiring.sources.slice(first); for (const stage of chain.stages) { if (stage.kind === 'envelope') this.envelope(wiring, stage.envelope).sources.push(...under); const impulse = stage.kind === 'reverb' && stage.impulse ? (wiring.buffers.get(stage.impulse) ?? null) : null; const made = stageNodes(context, stage, follow, wiring.at, impulse); tail.connect(made.input); tail = made.output; knobs.push(...made.knobs); if (made.stop) wiring.drivers.push({ start: () => {}, stop: made.stop }); if (stage.kind === 'tap') wiring.drivers.push({ start: () => {}, stop: this.spectra.hold(stage, made.output) }); } // A tap does nothing to the sound: the stages counted are those that do. wiring.stages += chain.stages.filter((stage) => stage.kind !== 'tap').length; return tail; } /** Sends the curve of an Envelope into a number of a voice, shaped to what that number holds (#335). */ private follower(wiring: Wiring): Follow { return (mod, param, ratio, map) => followCurve(wiring.context, this.envelope(wiring, mod.envelope).driver, mod, param, ratio, map); } /** The run of an Envelope in a voice, started on its first use: one curve however much it moves. */ private envelope(wiring: Wiring, envelope: FxEnvelope): EnvelopeRun { const known = wiring.envelopes.get(envelope.id); if (known) return known; const driver = envelopeDriver(wiring.context, envelope, wiring.at); wiring.drivers.push(driver); const run: EnvelopeRun = { envelope, driver, sources: [], released: false }; wiring.envelopes.set(envelope.id, run); return run; } /** * A way's sound: the pitch worklet where the way has a Pitch, which keeps the length whatever the * pitch does (#317), and otherwise the buffer as it is. Without the worklet a pitched way plays at * its own pitch rather than not at all. A way with a source of its own (#331) plays that file, at * its volume, and can fade out by itself when its note ends. */ private source(wiring: Wiring, chain: Way): AudioNode { const sound = chain.chip ? this.chip(wiring, chain, chain.chip) : this.sound(wiring, chain); const own = chain.source; if (!wiring.rings && !own && !chain.chip && !wiring.clip) return sound; const fade = gainNode(wiring.context, own?.volume ?? 1); wiring.sources.at(-1)!.fade = fade; return sound.connect(fade); } private sound(wiring: Wiring, way: Way): AudioNode { const { pitch, shifted, source: own, pitchMods } = way; if (!own && wiring.stream && this.hasWorklet) return this.streamSource( wiring, wiring.stream, playedPlan(wiring.stream.frames, wiring.offset, wiring.loop), way, ); const { context } = wiring; const file = own?.file ?? wiring.file; const loop = own ? own.loop : wiring.loop; const gate = own?.gate ?? false; // A file that could not be loaded leaves its way silent rather than the whole note. const buffer = wiring.buffers.get(file) ?? context.createBuffer(1, 1, context.sampleRate); wiring.pitches.push(pitch); if (shifted && this.hasWorklet) { const options: PitchOptions = { file, step: buffer.sampleRate / context.sampleRate, loop, startAt: wiring.at, }; if (wiring.offset > 0) options.skip = Math.round((wiring.offset % buffer.duration) * buffer.sampleRate); if (own && loop) Object.assign(options, loopIn(buffer, own.loopStart, own.loopEnd)); if (!this.sent.has(file)) { options.channels = Array.from({ length: buffer.numberOfChannels }, (_, c) => buffer.getChannelData(c), ); this.sent.add(file); } const node = new AudioWorkletNode(context, 'pitch-player', { numberOfInputs: 0, outputChannelCount: [buffer.numberOfChannels], parameterData: { pitch }, processorOptions: options, }); wiring.vocoders += 1; wiring.sources.push({ start: () => {}, stop: () => stopProcessor(node), ended: (then) => endedProcessor(node, then), fade: null, gate, leave: own && loop ? () => node.port.postMessage('leave') : null, }); const knob = node.parameters.get('pitch')!; wiring.knobs.push({ param: knob, target: pitch, even: 'value' }); // On top of the pitch the way has anyway, which a parameter can still move. const follow = this.follower(wiring); for (const mod of pitchMods ?? []) follow(mod, knob, false); return node; } const source = context.createBufferSource(); source.buffer = buffer; source.loop = loop; if (own && loop) { source.loopStart = own.loopStart; source.loopEnd = own.loopEnd; } wiring.sources.push({ // Web Audio holds an offset past the end of a looping buffer to its end, not round it. start: (at, offset) => source.start(at, offset > 0 ? offset % buffer.duration : 0), stop: () => source.stop(), ended: (then) => { source.onended = then; }, fade: null, gate, // Web Audio plays a buffer that stops looping on to its end, past `loopEnd`. leave: own && loop ? () => { source.loop = false; } : null, }); if (shifted) wiring.knobs.push({ param: null, target: pitch, even: 'value' }); return source; } /** * A way streamed from its copy (#349): a `stream-player` fed along its plan. Its Pitch is a shifter * of what it plays, as on a bus (#346): its length stays, and it is heard a frame late. */ private streamSource( wiring: Wiring, file: StreamFile, plan: StreamPlan, { pitch, shifted, pitchMods }: Way, ): AudioNode { const { context } = wiring; wiring.pitches.push(pitch); const { node, source } = this.streamNode(context, file, plan, wiring.at); wiring.sources.push(source); if (!shifted) return node; const shifter = new AudioWorkletNode(context, 'pitch-shifter', { numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [2], channelCount: 2, channelCountMode: 'explicit', parameterData: { pitch }, }); node.connect(shifter); wiring.vocoders += 1; wiring.drivers.push({ start: () => {}, stop: () => stopProcessor(shifter) }); const knob = shifter.parameters.get('pitch')!; wiring.knobs.push({ param: knob, target: pitch, even: 'value' }); const follow = this.follower(wiring); for (const mod of pitchMods ?? []) follow(mod, knob, false); return shifter; } /** A `stream-player` fed along `plan`, heard from `at` on, and the source a voice holds it by. */ private streamNode( context: BaseAudioContext, file: StreamFile, plan: StreamPlan, at: number, ): { node: AudioWorkletNode; source: Source; feed: StreamFeed } { let then: (() => void) | null = null; const feed: StreamFeed = new StreamFeed(file, plan, () => { this.dropFeed(feed); then?.(); }); const options: StreamOptions = { channels: file.channels, startAt: at, first: feed.opening() }; const node = new AudioWorkletNode(context, 'stream-player', { numberOfInputs: 0, outputChannelCount: [file.channels], processorOptions: options, }); feed.attach(node); this.feeds.add(feed); const source: Source = { start: () => {}, stop: () => { this.dropFeed(feed); feed.stop(); }, ended: (call) => { then = call; }, fade: null, gate: false, leave: null, }; return { node, source, feed }; } /** * Streams a name along `plan` into an offline context at `STREAM_RATE`, heard from `at`, `frames` * long, with the player and the feed a game has (test hook). The clock is held every half second * until the feed has caught up: offline it runs faster than anything is decoded. Null where the * name does not stream. */ async renderStream( name: string, plan: StreamPlan, at: number, frames: number, ): Promise<{ channels: Float32Array[]; skipped: number } | null> { const file = this.streamed(name); if (!file || !(await file.ready())) return null; const context = new OfflineAudioContext(file.channels, frames, STREAM_RATE); await context.audioWorklet.addModule(fxWorkletUrl); const { node, source, feed } = this.streamNode(context, file, plan, at); node.connect(context.destination); // Where the plan starts past the head, the first stretch is decoded before the clock runs. while (feed.busy) await new Promise((resolve) => setTimeout(resolve, 5)); const step = STREAM_RATE / 2; const hold = (frame: number) => { if (frame >= frames) return; void context.suspend(frame / STREAM_RATE).then(async () => { do await new Promise((resolve) => setTimeout(resolve, 5)); while (feed.busy); hold(frame + step); void context.resume(); }); }; hold(step); const rendered = await context.startRendering(); const skipped = feed.skipped; source.stop(); return { channels: Array.from({ length: rendered.numberOfChannels }, (_, c) => rendered.getChannelData(c)), skipped, }; } /** A library of its own, served from where this one is, that has decoded nothing yet (test hook). */ aside(): SoundLibrary { const library = new SoundLibrary(this.guarded); library.configure(this.base); return library; } /** All of the copy a name streams from, decoded as the feed decodes it (test hook). */ async streamedWhole(name: string): Promise { const file = this.streamed(name); if (!file || !(await file.ready())) return null; return decodeSpan(file, 0, file.frames); } private dropFeed(feed: StreamFeed): void { if (!this.feeds.delete(feed)) return; this.skippedBefore += feed.skipped; } /** * A Chip (#329): a worklet of its own that ends with the note. A Pitch on its way moves its * frequency itself. Without the worklet the way is silent and ends at once. */ private chip(wiring: Wiring, { pitch, shifted, pitchMods }: Way, chip: FxChip): AudioNode { const { context, knobs } = wiring; wiring.pitches.push(pitch); const { frequency, duty, frequencyMod, dutyMod, ...shape } = chip; if (!this.hasWorklet) { const silent = context.createBufferSource(); silent.buffer = context.createBuffer(1, 1, context.sampleRate); wiring.sources.push({ start: (at) => silent.start(at), stop: () => silent.stop(), ended: (then) => { silent.onended = then; }, fade: null, gate: true, leave: null, }); if (shifted) knobs.push({ param: null, target: pitch, even: 'value' }); knobs.push( { param: null, target: frequency, even: 'ratio' }, { param: null, target: duty, even: 'value' }, ); return silent; } const options: ChipOptions = { ...shape, startAt: wiring.at }; const node = new AudioWorkletNode(context, 'chip-player', { numberOfInputs: 0, outputChannelCount: [1], parameterData: { frequency: frequencyMod ? 0 : frequency, duty: dutyMod ? 0 : duty, pitch }, processorOptions: options, }); wiring.sources.push({ start: () => {}, stop: () => stopProcessor(node), ended: (then) => endedProcessor(node, then), fade: null, gate: true, leave: null, }); const param = (name: string) => node.parameters.get(name)!; if (shifted) knobs.push({ param: param('pitch'), target: pitch, even: 'value' }); knobs.push( { param: frequencyMod ? null : param('frequency'), target: frequency, even: 'ratio' }, { param: dutyMod ? null : param('duty'), target: duty, even: 'value' }, ); const follow = this.follower(wiring); if (frequencyMod) follow(frequencyMod, param('frequency'), true); if (dutyMod) follow(dutyMod, param('duty'), false); for (const mod of pitchMods ?? []) follow(mod, param('pitch'), false); return node; } /** * Sends every knob of a voice towards what its values say now, in `time` seconds of the audio * clock along `ease`, or else in a quick glide. Says whether any of them had somewhere to go. */ private glide(voice: Voice, time: number, ease: EaseShare): boolean { const levels: number[] = []; for (const plan of voice.fx.layers) levelsOf(fxChainOf(plan, voice.fx.params, voice.fx.tempo), levels); return glideKnobs(voice.knobs, levels, this.audio().currentTime, time, ease); } private audio(): BaseAudioContext { if (this.context) return this.context; const context = new AudioContext(); this.context = context; if (this.guarded) this.guardOut(context); if (this.mix.length > 0) void this.applyMix(); return context; } private guardOut(context: AudioContext): void { const guard: NonNullable = { input: context.createGain(), limiter: null, latency: 0, }; this.guard = guard; guard.input.connect(context.destination); void this.fxWorklet().then((loaded) => { if (!loaded || this.guard !== guard) return; let limiter: AudioWorkletNode; try { limiter = new AudioWorkletNode(context, 'limiter', { outputChannelCount: [2], channelCount: 2, channelCountMode: 'explicit', parameterData: { threshold: GUARD_DB, ceiling: GUARD_DB, release: GUARD_RELEASE, autoGain: 0 }, }); } catch { return; } guard.input.disconnect(); guard.input.connect(limiter).connect(context.destination); guard.limiter = limiter; guard.latency = limiterLatency(context.sampleRate); }); } /** * What a name plays: the copy of a clip (#349) once it is decoded, else a cut made from its file the * first time; null while neither is loaded. */ private buffer(name: string): AudioBuffer | null { const whole = this.buffers.get(name); if (whole) return whole; const copy = this.copyOf(name); const copied = copy ? this.buffers.get(copy) : undefined; if (copied) return copied; const held = this.cuts.get(name); if (held) { this.cuts.delete(name); this.cuts.set(name, held); return held; } const range = this.rangeOf(name); if (!range) return null; const made = this.decoder(name).createBuffer( range.whole.numberOfChannels, range.to - range.from, range.whole.sampleRate, ); const channels = Array.from({ length: made.numberOfChannels }, (_, c) => range.whole.getChannelData(c)); cutChannels(channels, range).forEach((data, c) => made.copyToChannel(data, c)); this.cuts.set(name, made); const samples = (buffer: AudioBuffer) => buffer.length * buffer.numberOfChannels; let total = [...this.cuts.values()].reduce((sum, buffer) => sum + samples(buffer), 0); for (const [other, buffer] of this.cuts) { if (total <= CUT_BUDGET || other === name) break; this.cuts.delete(other); total -= samples(buffer); } return made; } /** * The samples of its loaded file a name plays: all of them, or the stretch a cut keeps – all of a * clip's copy (#349), which is that stretch already. */ private rangeOf(name: string): (CutRange & { whole: AudioBuffer }) | null { const { file, cut } = parseCutName(name); const copy = this.copyOf(name); const copied = copy ? this.buffers.get(copy) : undefined; const whole = copied ?? this.buffers.get(file); if (!whole) return null; if (!cut || copied) return { whole, from: 0, to: whole.length, fadeIn: 0, fadeOut: 0, curveIn: 'linear', curveOut: 'linear', reverse: false, }; return { whole, ...cutRange(whole.length, whole.sampleRate, cut) }; } /** How long a sound a Reverb takes as its impulse response rings (#328): all it may, while unknown. */ private readonly soundLength = (file: string): number => this.duration(file) ?? REVERB_MAX_DECAY; /** Whether the file of a name could not be had: a cut of it cannot either, unless it has a copy. */ private hasFailed(name: string): boolean { return this.copyOf(name) === null && this.failed.has(parseCutName(name).file); } /** Loads the FX worklet (Pitch, Bitcrusher, Chip, Phaser, Flanger) into the context once; says whether it is there. */ private fxWorklet(): Promise { return (this.workletLoading ??= (async () => { try { await this.audio().audioWorklet.addModule(fxWorkletUrl); return (this.hasWorklet = true); } catch (error: unknown) { console.error('[sounds] cannot load the FX worklet, its effects and Chips are not heard', error); return (this.hasWorklet = false); } })()); } /** * The context that is already playing, or else one that decodes without opening a device. An Ogg * file, which a bake (#340) is, is decoded at the rate Opus is made at, whatever the device runs * at: resampled, the loop of a bake would lose a sample a round. */ private decoder(file: string): BaseAudioContext { if (typeof OfflineAudioContext === 'undefined') return this.audio(); if (file.endsWith('.ogg')) return (this.opusContext ??= new OfflineAudioContext(1, 1, BAKE_RATE)); if (this.context) return this.audio(); return (this.decodeContext ??= new OfflineAudioContext(1, 1, DECODE_RATE)); } /** Everything that would have silenced this file for this owner, counted rather than flagged. */ private cancelsFor(file: string, owner: string): number { return (this.stops.get(file) ?? 0) + this.allStops + (this.ownerStops.get(owner) ?? 0); } } /** * What a library that bakes (#340) runs on: an offline context, the library whose decoded files it * plays, and what wakes its schedulers – the offline clock, not the page's timers. */ export interface OfflineRender { context: OfflineAudioContext; files: SoundLibrary; wake(pump: () => void): () => void; } /** Seconds a bake fades out over where its notes would have rung on (#340). */ const BAKE_LET_RING = 0.3; /** The bus of a game's playing (#336): its master for `''`, else the track of that name inside it. */ function groupBus(id: number, track: string): string { return track === '' ? `play:${id}` : `play:${id}:${track}`; } /** * A track of a song (#334): what goes in passes its sound FX (#346), then its level, then its pan, * then the bus it goes into or straight out – or each of its sends (#350); `set` is what level and * pan glide to. */ interface Bus { input: GainNode; gain: GainNode; pan: StereoPannerNode; set: { gain: number; pan: number }; /** Where it goes as a whole while it has no sends; null straight out. */ into: string | null; sends: BusSendNodes[] | null; /** The sound FX it is set to, and the chain made of it once its worklet is there. */ fx: FxVoice | null; chain: BusChain | null; /** Set while it is to be let go a moment later, unless it is used again first. */ leaving: number; /** Its level meter (#359), while one is asked after. */ meter: BusMeter | null; } /** A tap after the pan of a bus: an analyser for each side, so a peak panned hard is not halved. */ interface BusMeter { split: ChannelSplitterNode; sides: AnalyserNode[]; samples: Float32Array; } /** One send of a track (#350): where it goes, where it taps, its level and how far it is held back. */ interface BusSendNodes { to: string; tap: SendTap; gain: number; delay: DelayNode; level: GainNode; } /** The sound FX of a bus as nodes (#346): one chain everything on the bus goes through. */ interface BusChain { /** What comes out of it, on to the level of the bus: where a send after the sound FX taps. */ out: AudioNode; /** In the order `levelsOf` lists the numbers of its chain. */ knobs: Knob[]; /** Seconds it is late: its pitch shifters, the other way of a Mix held back to match them. */ latency: number; stops: (() => void)[]; nodes: AudioNode[]; } /** What `stats` says about the last voice: enough for a check to tell which effects it went through. */ interface LastVoice { file: string; volume: number; loop: boolean; /** How many layers of a note it played side by side (#331); 1 for a sound. */ layers: number; /** Semitones of each way (#316), and how many gain and filter nodes it passed. */ pitches: number[]; stages: number; /** Seconds of what was played: a pitched voice is as long as the sound itself (#312, #317). */ duration: number; /** When it started on the audio clock, what a check times it from. */ at: number; /** How many Envelopes shape it (#335). */ envelopes: number; /** The bus it went into, `''` for the master (#325). */ bus: string; } /** The stages that run on the worklet. */ const ON_WORKLET: ReadonlySet = new Set([ 'crush', 'phaser', 'flanger', 'limit', 'tremolo', 'vibrato', ]); /** Whether a plan has a Pitch, a Chip or a stage of the worklet anywhere, and so needs the worklet. */ function needsWorklet(plan: FxPlan): boolean { if (plan.stages.some(onWorklet)) return true; return 'mix' in plan ? needsWorklet(plan.a) || needsWorklet(plan.b) : plan.pitch.length > 0 || !!plan.chip; } const onWorklet = (stage: FxStagePlan): boolean => stage.kind === 'wet' ? stage.stages.some(onWorklet) : ON_WORKLET.has(stage.kind); /** Whether a way of a plan plays a Chip (#329), which needs no file. */ function hasChip(plan: FxPlan): boolean { return 'mix' in plan ? hasChip(plan.a) || hasChip(plan.b) : plan.chip !== undefined; } /** The files the ways of a plan read with a source of their own (#331). */ function filesOf(plan: FxPlan): string[] { if ('mix' in plan) return [...filesOf(plan.a), ...filesOf(plan.b)]; return plan.source ? [plan.source.file] : []; } /** * Where a loop runs in the samples of a sound, as the worklet takes it: between `start` and `end` * seconds, 0 at the end being the end of the sound. One that does not fit is the whole sound, as * Web Audio has it for a buffer source. */ function loopIn(buffer: AudioBuffer, start: number, end: number): { loopStart: number; loopEnd: number } { const last = buffer.length; const from = Math.round(Math.max(0, start) * buffer.sampleRate); const to = end > 0 ? Math.min(last, Math.round(end * buffer.sampleRate)) : last; return from < to ? { loopStart: from, loopEnd: to } : { loopStart: 0, loopEnd: last }; } /** The loudest sample of each of `PEAK_BUCKETS` equal stretches of what is heard of `whole`, at most 1. */ function peaksOf( whole: AudioBuffer, { from, to, reverse }: Pick, ): Float32Array { const peaks = new Float32Array(PEAK_BUCKETS); for (let c = 0; c < whole.numberOfChannels; c++) { const data = whole.getChannelData(c); for (let i = from; i < to; i++) { const heard = reverse ? to - 1 - i : i - from; const bucket = Math.floor((heard / (to - from)) * PEAK_BUCKETS); peaks[bucket] = Math.max(peaks[bucket]!, Math.abs(data[i]!)); } } for (let b = 0; b < PEAK_BUCKETS; b++) peaks[b] = Math.min(1, peaks[b]!); return peaks; } /** The loudest sample of each side over the stretch a meter looks back over, 1 at full scale. */ function readMeter(meter: BusMeter): [number, number] { const [left, right] = meter.sides.map((side) => { side.getFloatTimeDomainData(meter.samples); let peak = 0; for (const sample of meter.samples) peak = Math.max(peak, Math.abs(sample)); return peak; }); return [left!, right!]; } export const soundLibrary = new SoundLibrary();