From 501a1524ed7294e47265244c2abff782f8e0b7ae Mon Sep 17 00:00:00 2001 From: MelekTaus Date: Mon, 5 Oct 2026 21:47:13 +0200 Subject: [PATCH] Ramp sound FX parameters over a time on the audio clock A pitch sweep counted down in On update or On fixed update runs in game time, while the sound runs on the clock of the audio device. A short sound magnifies any offset between the two, so the sweep came out different on every play and most of all on the first. - Set sound FX parameters has a new input time in seconds. The values run evenly to their target in that time, scheduled on the audio clock with linearRampToValueAtTime or exponentialRampToValueAtTime. - Evenly in their own unit: a pitch in semitones and a gain in decibels, an EQ band and a Mix share linearly in their value. - A time of 0 or below keeps the quick glide of about 0.1 s. - Each change starts from where the voice is now. Whatever was still planned is dropped, so a change never queues behind an older one. Firefox lacks cancelAndHoldAtTime, there the value is held by hand. - A voice still waiting on its bytes starts a timed change with the sound, so the whole sweep is heard. Tests: - node-catalog: time is handed on, a time below 0 is a quick glide. - sound-library: semitones and decibels ramp in their own steps, a timed change on a waiting voice starts with the sound. - sound-library + node-catalog + soundfx: 419 tests, green. - e2e 34 on melektaus-atx: ALL PASSED, 31 checks. A 2 s ramp from depth 7 to -5 read speed 0.979, 0.926 and 0.778 half a second on. - Counter-check: folding a timed change on a waiting voice into its start values fails the new test. - The gate did not run. Co-Authored-By: Claude Opus 5.5 --- src/core/graph/NodeContext.ts | 5 +- .../nodes/action/SetSoundFxParameters.ts | 9 ++- src/runtime/systems/Audio.ts | 6 +- src/runtime/systems/behaviorContext.ts | 2 +- src/viewport/SoundLibrary.ts | 67 ++++++++++++++----- tests/core/node-catalog.test.ts | 16 ++++- tests/helpers.ts | 4 +- tests/viewport/sound-library.test.ts | 56 +++++++++++++++- tools/e2e/34-sound.mjs | 18 +++++ 9 files changed, 154 insertions(+), 29 deletions(-) diff --git a/src/core/graph/NodeContext.ts b/src/core/graph/NodeContext.ts index 516043a0..c6b9927d 100644 --- a/src/core/graph/NodeContext.ts +++ b/src/core/graph/NodeContext.ts @@ -254,9 +254,10 @@ export interface NodeContext { stopSound(assetId: string): void; /** * Hands new values to the sound FX parameters of one voice, by the number `playSound` gave it - * (#315), so a running sound follows the game. A voice that has ended is not reached. + * (#315), so a running sound follows the game. A voice that has ended is not reached. `time` is + * how many seconds the values take to get there, on the audio clock; 0 is a quick glide. */ - tuneSound(voice: number, fxParams: FxParams): void; + tuneSound(voice: number, fxParams: FxParams, time: number): void; /** * Asks for another scene, by asset id. The swap happens once the frame is over – a graph must be * able to finish the flow it is in, and half a frame in the old world and half in the new one is diff --git a/src/core/graph/nodes/action/SetSoundFxParameters.ts b/src/core/graph/nodes/action/SetSoundFxParameters.ts index 5c9c3fdc..bf265efe 100644 --- a/src/core/graph/nodes/action/SetSoundFxParameters.ts +++ b/src/core/graph/nodes/action/SetSoundFxParameters.ts @@ -16,17 +16,22 @@ export const SetSoundFxParameters: NodeDef = { 'Changes the sound FX parameters of a sound while it plays: the voice a Play sound handed out ' + 'glides to the values from a Sound FX parameters node. Keep the voice in a variable to reach it ' + 'from another event. A parameter not given keeps its value, a voice that has ended is not ' + - 'reached. A pitch changes by speed here, so the sound gets shorter or longer while it is moved.', + 'reached. A pitch changes by speed here, so the sound gets shorter or longer while it is moved. ' + + 'time: seconds the values take to get there, evenly and on the clock of the sound itself, so ' + + 'a sweep sounds the same every time; 0 is a quick glide.', inputs: [ exec(), { ...num('voice'), inline: false }, { key: 'fxParams', label: 'fx params', type: 'fxparams' }, + num('time'), ], outputs: [exec()], params: [], run: (ctx, inputs) => { const voice = asNumber(inputs.voice); - if (voice > 0) ctx.tuneSound(voice, fxParamsOf(inputs.fxParams)); + const time = asNumber(inputs.time); + if (voice > 0) + ctx.tuneSound(voice, fxParamsOf(inputs.fxParams), Number.isFinite(time) && time > 0 ? time : 0); return { next: 'exec' }; }, }; diff --git a/src/runtime/systems/Audio.ts b/src/runtime/systems/Audio.ts index b2b6a77e..ff8d0ce9 100644 --- a/src/runtime/systems/Audio.ts +++ b/src/runtime/systems/Audio.ts @@ -21,9 +21,11 @@ export interface AudioSink { /** * Hands new values to the sound FX parameters of one voice, by the number `play` gave it (#315). * A name its FX does not have is left out, one not given keeps its value. A voice that has ended - * is not there any more and nothing happens – a detached one is reached like any other. + * is not there any more and nothing happens – a detached one is reached like any other. `time` is + * how many seconds the values take to get there, counted on the audio clock from now – or from + * the moment the voice really starts, where it still waits on its bytes. 0 is a quick glide. */ - tune(voice: number, params: FxParams): void; + tune(voice: number, params: FxParams, time?: number): void; /** Silences every voice of that file; `''` silences all of them. */ stop(file: string): void; /** Silences every voice started by that owner – what destroying a behavior does. */ diff --git a/src/runtime/systems/behaviorContext.ts b/src/runtime/systems/behaviorContext.ts index bce400fe..46e4d259 100644 --- a/src/runtime/systems/behaviorContext.ts +++ b/src/runtime/systems/behaviorContext.ts @@ -172,7 +172,7 @@ export function behaviorContext( const file = assetId === '' ? '' : soundFileOf(project, assetId); if (assetId === '' || file) run.audio.stop(file); }, - tuneSound: (voice, fxParams) => run.audio.tune(voice, fxParams), + tuneSound: (voice, fxParams, time) => run.audio.tune(voice, fxParams, time), spawn: (assetId, position, parent) => world.spawn(assetId, position, resolve(parent) ?? null), addJoint: (assetId, owner, connected, at) => { const on = resolve(owner); diff --git a/src/viewport/SoundLibrary.ts b/src/viewport/SoundLibrary.ts index b1c3485b..1e33c08c 100644 --- a/src/viewport/SoundLibrary.ts +++ b/src/viewport/SoundLibrary.ts @@ -20,9 +20,11 @@ const DECODE_RATE = 48000; /** * One number of a voice that can still change while it plays (#315): a level, a share of a Mix, the * gain of a band – or the pitch of a source, which glides by speed from the pitch its copy was made at. + * `even` says what a timed change runs evenly in: a share or a band in what it holds, a gain stage in + * decibels (its node holds the factor), a pitch in semitones (its node holds the speed). */ type Knob = - | { kind: 'level'; param: AudioParam; target: number } + | { kind: 'level'; param: AudioParam; target: number; even: 'value' | 'ratio' } | { kind: 'pitch'; source: AudioBufferSourceNode; base: number; target: number }; /** @@ -53,10 +55,14 @@ interface Wiring { stages: number; } -/** A play still waiting on its bytes: what a `tune` in the meantime writes into. */ +/** + * A play still waiting on its bytes: what a quick `tune` in the meantime writes into, and the timed + * ones, which have to start with the voice rather than run out before it is heard. + */ interface Pending { id: number; fx: FxVoice; + timed: { params: FxParams; time: number }[]; } /** @@ -122,29 +128,33 @@ export class SoundLibrary implements AudioSink { return id; } const askedAt = this.cancelsFor(file, owner); - const waiting: Pending = { id, fx: { plan: fx.plan, params: { ...fx.params } } }; + const waiting: Pending = { id, fx: { plan: fx.plan, params: { ...fx.params } }, timed: [] }; this.pending.push(waiting); void this.load(file).then((loaded) => { this.pending = this.pending.filter((other) => other !== waiting); // A `Stop sound` – or the destruction of the behavior that asked – that happened while the // bytes were in flight has to win, or a sound the run already ended would start afterwards. - if (loaded && this.cancelsFor(file, owner) === askedAt) - this.start(id, file, loaded, volume, loop, owner, waiting.fx); + if (!loaded || this.cancelsFor(file, owner) !== askedAt) return; + this.start(id, file, loaded, volume, loop, owner, waiting.fx); + for (const { params, time } of waiting.timed) this.tune(id, params, time); }); return id; } /** Says whether the voice was still there to be reached – playing, or waiting on its bytes. */ - tune(id: number, params: FxParams): boolean { + tune(id: number, params: FxParams, time = 0): boolean { const waiting = this.pending.find((other) => other.id === id); if (waiting) { - waiting.fx = { plan: waiting.fx.plan, params: { ...waiting.fx.params, ...params } }; + // A quick change is where the voice starts; a timed one – and anything after it, so the order + // holds – runs from the moment it is heard. + if (time > 0 || waiting.timed.length > 0) waiting.timed.push({ params, time }); + else waiting.fx = { plan: waiting.fx.plan, params: { ...waiting.fx.params, ...params } }; return true; } const voice = this.voices.find((other) => other.id === id); if (!voice) return false; voice.fx = { plan: voice.fx.plan, params: { ...voice.fx.params, ...params } }; - this.glide(voice); + this.glide(voice, time); this.tuned += 1; return true; } @@ -308,16 +318,21 @@ export class SoundLibrary implements AudioSink { const wayA = this.wire(wiring, chain.a); const shareA = gainNode(context, 1 - chain.mix); wayA.connect(shareA).connect(tail); - knobs.push({ kind: 'level', param: shareA.gain, target: 1 - chain.mix }); + knobs.push({ kind: 'level', param: shareA.gain, target: 1 - chain.mix, even: 'value' }); const wayB = this.wire(wiring, chain.b); const shareB = gainNode(context, chain.mix); wayB.connect(shareB).connect(tail); - knobs.push({ kind: 'level', param: shareB.gain, target: chain.mix }); + knobs.push({ kind: 'level', param: shareB.gain, target: chain.mix, even: 'value' }); } else tail = this.source(wiring, chain.pitch); for (const stage of chain.stages) { const [node, param] = fxNode(context, stage); tail = tail.connect(node); - knobs.push({ kind: 'level', param, target: stage.gain }); + knobs.push({ + kind: 'level', + param, + target: stage.gain, + even: stage.kind === 'gain' ? 'ratio' : 'value', + }); } wiring.stages += chain.stages.length; return tail; @@ -334,10 +349,11 @@ export class SoundLibrary implements AudioSink { } /** - * Sends every knob of a voice towards what its values say now. A pitch cannot get a new copy in - * the middle of a sound, so it glides by speed: the voice gets that much shorter or longer. + * Sends every knob of a voice towards what its values say now, in `time` seconds of the audio + * clock or else in a quick glide. A pitch cannot get a new copy in the middle of a sound, so it + * glides by speed: the voice gets that much shorter or longer. */ - private glide(voice: Voice): void { + private glide(voice: Voice, time: number): void { const levels = levelsOf(fxChainOf(voice.fx.plan, voice.fx.params)); const now = this.audio().currentTime; voice.knobs.forEach((knob, i) => { @@ -345,8 +361,8 @@ export class SoundLibrary implements AudioSink { if (level === undefined || level === knob.target) return; knob.target = level; if (knob.kind === 'pitch') - knob.source.playbackRate.setTargetAtTime(2 ** ((level - knob.base) / 12), now, GLIDE); - else knob.param.setTargetAtTime(level, now, GLIDE); + move(knob.source.playbackRate, 2 ** ((level - knob.base) / 12), now, time, true); + else move(knob.param, level, now, time, knob.even === 'ratio'); }); } @@ -427,6 +443,25 @@ function fxNode(context: AudioContext, stage: FxStage): [AudioNode, AudioParam] return [filter, filter.gain]; } +/** + * Sends one knob from where it is right now to `to`. Whatever was still planned for it is dropped + * first, so a change never queues behind an older one. A timed change runs evenly – a `ratio` one + * evenly in its logarithm, which is what decibels and semitones are. + */ +function move(param: AudioParam, to: number, now: number, time: number, ratio: boolean): void { + // Firefox has no cancelAndHoldAtTime: there the value it reads out now is held by hand. + const held = param as Omit & + Partial>; + if (held.cancelAndHoldAtTime) held.cancelAndHoldAtTime(now); + else { + param.cancelScheduledValues(now); + param.setValueAtTime(param.value, now); + } + if (time <= 0) param.setTargetAtTime(to, now, GLIDE); + else if (ratio && to > 0) param.exponentialRampToValueAtTime(to, now + time); + else param.linearRampToValueAtTime(to, now + time); +} + /** The numbers of a chain in the order `wire` makes their knobs: a stage's `gain` is what its node holds. */ function levelsOf(chain: FxChain, into: number[] = []): number[] { if ('mix' in chain) { diff --git a/tests/core/node-catalog.test.ts b/tests/core/node-catalog.test.ts index bdbc037b..e2917367 100644 --- a/tests/core/node-catalog.test.ts +++ b/tests/core/node-catalog.test.ts @@ -2032,7 +2032,19 @@ const CASES: Record = { name: 'hands the numbers of the bundle to the voice it is given', inputs: { voice: 4, fxParams: { depth: 3, label: 'x' } }, next: 'exec', - after: (ctx) => expect(ctx.tuned).toEqual([{ voice: 4, fxParams: { depth: 3 } }]), + after: (ctx) => expect(ctx.tuned).toEqual([{ voice: 4, fxParams: { depth: 3 }, time: 0 }]), + }, + { + name: 'hands on the time the values take to get there', + inputs: { voice: 4, fxParams: { depth: -64 }, time: 1.28 }, + next: 'exec', + after: (ctx) => expect(ctx.tuned).toEqual([{ voice: 4, fxParams: { depth: -64 }, time: 1.28 }]), + }, + { + name: 'a time below 0 is a quick glide', + inputs: { voice: 4, fxParams: { depth: 3 }, time: -2 }, + next: 'exec', + after: (ctx) => expect(ctx.tuned).toEqual([{ voice: 4, fxParams: { depth: 3 }, time: 0 }]), }, { name: 'voice 0 – nothing was played – reaches nothing, and the flow carries on', @@ -2119,6 +2131,7 @@ const EVENT_NODES: Record = { OnStart: 'graph.test.ts', OnEnable: 'behavior-events.test.ts', OnUpdate: 'graph.test.ts', + OnFixedUpdate: 'fixed-update.test.ts', OnDisable: 'behavior-events.test.ts', OnDestroy: 'behavior-events.test.ts', OnKeyDown: 'behavior-events.test.ts', @@ -2131,7 +2144,6 @@ const EVENT_NODES: Record = { OnClick: 'nodes.test.ts', OnMouseWheel: 'clip-areas.test.ts', OnAnimationFinished: 'animation.test.ts', - OnFixedUpdate: 'fixed-update.test.ts', OnStateEntered: 'state-machine.test.ts', OnStateExited: 'state-machine.test.ts', }; diff --git a/tests/helpers.ts b/tests/helpers.ts index d65527b9..082ffe11 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -214,7 +214,7 @@ export type FakeContext = NodeContext & { fxParams: Readonly>; }[]; silenced: string[]; - tuned: { voice: number; fxParams: Readonly> }[]; + tuned: { voice: number; fxParams: Readonly>; time: number }[]; /** The scenes the context was asked to load, in order. */ loaded: string[]; /** The time scale and the pause the context holds, as the game nodes left them. */ @@ -271,7 +271,7 @@ export function fakeContext(overrides: Partial = {}): FakeContext { ctx.played.push({ assetId, volume, loop, detached, fx, fxParams }), stopSound: (assetId) => ctx.silenced.push(assetId), tuned: [], - tuneSound: (voice, fxParams) => ctx.tuned.push({ voice, fxParams }), + tuneSound: (voice, fxParams, time) => ctx.tuned.push({ voice, fxParams, time }), loadScene: (sceneId) => ctx.loaded.push(sceneId), spawn: () => null, addJoint: () => null, diff --git a/tests/viewport/sound-library.test.ts b/tests/viewport/sound-library.test.ts index 1a14cede..565cb66c 100644 --- a/tests/viewport/sound-library.test.ts +++ b/tests/viewport/sound-library.test.ts @@ -12,15 +12,35 @@ import { SoundLibrary } from '../../src/viewport/SoundLibrary'; const FILE = 'a.wav'; const OTHER = 'b.wav'; -/** An AudioParam that remembers where it was sent to glide, and how often. */ +/** + * An AudioParam that remembers where it was sent to glide, where it was sent to ramp – "linear" or + * "exponential", to what, by when – and how often it was held. It has no cancelAndHoldAtTime, as in + * Firefox, so the hold by hand is what runs here. + */ interface FakeParam { value: number; glides: number[]; + ramps: [string, number, number][]; + holds: number; setTargetAtTime: (value: number) => void; + cancelScheduledValues: () => void; + setValueAtTime: () => void; + linearRampToValueAtTime: (value: number, at: number) => void; + exponentialRampToValueAtTime: (value: number, at: number) => void; } function param(value: number): FakeParam { - const made: FakeParam = { value, glides: [], setTargetAtTime: (to) => made.glides.push(to) }; + const made: FakeParam = { + value, + glides: [], + ramps: [], + holds: 0, + setTargetAtTime: (to) => made.glides.push(to), + cancelScheduledValues: () => {}, + setValueAtTime: () => void (made.holds += 1), + linearRampToValueAtTime: (to, at) => made.ramps.push(['linear', to, at]), + exponentialRampToValueAtTime: (to, at) => made.ramps.push(['exponential', to, at]), + }; return made; } @@ -401,6 +421,38 @@ describe('the sound library', () => { expect(gains.map((g) => g.value)).toContain(0.5); expect(sources[0]!.playbackRate.glides).toEqual([]); }); + + it('runs a timed change evenly on the audio clock: semitones and decibels in their own steps', async () => { + const library = new SoundLibrary(); + const voice = await play(library, FILE, { owner: 'me' }, { plan: open, params: {} }); + const [, , shareA, loud, shareB] = gains; + library.tune(voice, { depth: 12, loud: -6, blend: 0.5 }, 1.5); + expect(sources[0]!.playbackRate.ramps).toEqual([['exponential', 2, 1.5]]); + expect(loud!.ramps).toEqual([['exponential', expect.closeTo(10 ** (-6 / 20)), 1.5]]); + expect([shareA!.ramps, shareB!.ramps]).toEqual([[['linear', 0.5, 1.5]], [['linear', 0.5, 1.5]]]); + expect([sources[0]!.playbackRate.glides, loud!.glides, shareA!.glides]).toEqual([[], [], []]); + // Each from where it is now: what was planned before is held, not queued behind. + expect([sources[0]!.playbackRate.holds, loud!.holds, shareA!.holds]).toEqual([1, 1, 1]); + + // A quick change after it holds the ramp where it is and glides on from there. + library.tune(voice, { depth: 0 }); + expect(sources[0]!.playbackRate.holds).toBe(2); + expect(sources[0]!.playbackRate.glides).toEqual([1]); + }); + + it('starts a timed change to a voice still waiting on its bytes when the voice starts', async () => { + const library = new SoundLibrary(); + const voice = await play(library, FILE, { hold: true, owner: 'me' }, { plan: open, params: {} }); + library.tune(voice, { depth: -64 }, 1.28); + // A quick change after a timed one keeps its place behind it. + library.tune(voice, { blend: 1 }); + expect(library.stats().started).toBe(0); + await arrive(); + // The voice is heard at its old pitch and only then sets off, so the whole sweep is heard. + expect(library.stats().last?.pitches).toEqual([0, 0]); + expect(sources[0]!.playbackRate.ramps).toEqual([['exponential', 2 ** (-64 / 12), 1.28]]); + expect(gains[4]!.glides).toEqual([1]); + }); }); it('a stop while the bytes are in flight keeps the sound from starting afterwards', async () => { diff --git a/tools/e2e/34-sound.mjs b/tools/e2e/34-sound.mjs index 3a635d9f..0455df14 100644 --- a/tools/e2e/34-sound.mjs +++ b/tools/e2e/34-sound.mjs @@ -474,6 +474,24 @@ export async function sound(t) { JSON.stringify({ last: tunedStats.last, tuned: tunedStats.tuned, levels, rate }), ); await stop(); + // A `time` of 2 s takes the same voice from depth 7 to -5 evenly on the audio clock: half a + // second in it is still well above the end (speed 0.5), where a quick glide would be there by then. + await mutate(` + const graph = __gm.state.project.assets.find((a) => a.id === 'sn-logic').graph; + graph.nodes.find((n) => n.id === 'sn-tune').params.time = 2; + graph.nodes.find((n) => n.id === 'sn-fxp2').params.depth = -5;`); + await play(); + const speed = async () => (await json(`__gm.sound.levels(${JSON.stringify(blip.file)})`))?.[0]?.[0]; + const early = await waitFor(speed, (s) => typeof s === 'number' && s < 0.99); + const later = await waitFor(speed, (s) => typeof s === 'number' && s < early - 0.05); + await new Promise((resolve) => setTimeout(resolve, 500)); + const afterHalf = await speed(); + check( + 'Set sound FX parameters with a time ramps the pitch over that time instead of jumping', + early > 0.6 && later > 0.55 && afterHalf > 0.52 && afterHalf < later, + JSON.stringify({ early, later, afterHalf }), + ); + await stop(); await mutate(` const logic = __gm.state.project.assets.find((a) => a.id === 'sn-logic'); logic.variables = [];