diff --git a/src/core/soundfx/nodes.ts b/src/core/soundfx/nodes.ts index b4eb0cfc..ed39df27 100644 --- a/src/core/soundfx/nodes.ts +++ b/src/core/soundfx/nodes.ts @@ -123,7 +123,7 @@ export const SoundFxOutput: SoundFxNodeDef = { emit: (plan) => plan, }; -const GAIN = numberRow('gain', 'Gain (dB)', -60, 24, 0.5); +const GAIN = numberRow('gain', 'Gain (dB)', -120, 24, 0.5); export const SoundFxGain: SoundFxNodeDef = { type: 'SoundFxGain', @@ -140,14 +140,14 @@ export const SoundFxGain: SoundFxNodeDef = { }, }; -const SEMITONES = numberRow('semitones', 'Semitones', -24, 24, 1); +const SEMITONES = numberRow('semitones', 'Semitones', -64, 64, 0.1); export const SoundFxPitch: SoundFxNodeDef = { type: 'SoundFxPitch', label: 'Pitch', category: 'sound', description: - 'Plays the sound higher or lower, in semitones: 12 is an octave up. It is the speed the sound is played at, so a higher one is also shorter, and it applies to the whole sound wherever the node sits in the line.', + 'Plays the sound higher or lower, in semitones: 12 is an octave up, 0.5 a quarter tone. The sound keeps its length, and the pitch applies to the whole sound wherever the node sits in the line.', inputs: [audioIn, numberIn(SEMITONES)], outputs: [audioOut], params: [], @@ -168,11 +168,11 @@ export const EQ_BANDS: readonly { key: string; label: string; filter: FxFilter; /** About an octave and a half wide: neighbouring peaks overlap, so no frequency falls between them. */ const PEAK_Q = 1; -const EQ_ROWS = EQ_BANDS.map((band) => numberRow(band.key, band.label, -24, 24, 0.5)); +const EQ_ROWS = EQ_BANDS.map((band) => numberRow(band.key, band.label, -60, 24, 0.5)); export const SoundFxEq: SoundFxNodeDef = { type: 'SoundFxEq', - label: 'EQ', + label: '5-Band-EQ', category: 'sound', description: 'A five-band equalizer: raises or lowers the lows (80 Hz), low mids (250 Hz), mids (1 kHz), high mids (4 kHz) and highs (12 kHz), each in decibels. A band at 0 does nothing.', diff --git a/src/core/soundfx/pitchShift.ts b/src/core/soundfx/pitchShift.ts new file mode 100644 index 00000000..bb723e15 --- /dev/null +++ b/src/core/soundfx/pitchShift.ts @@ -0,0 +1,95 @@ +/** Half a frame, in seconds: the hop between frames. A frame is longer than the period of a low note. */ +const HOP_SECONDS = 0.02; +/** How far a frame may move to fit the one before it: a period of 80 Hz. */ +const REACH_SECONDS = 0.0125; +/** The first search looks at every fourth offset and sample, the second at every one around the best. */ +const COARSE = 4; + +/** + * One channel of a sound played `semitones` higher or lower at its own length (#312). The sound is + * first read faster or slower, which sets the pitch and gets the length wrong, and then stretched + * back by WSOLA: frames half a frame apart are laid over each other under a Hann window, each one + * moved a little to where it continues the frame before it, so a tone carries on without a jump in + * phase. A sound that loops reads on across its end into its start, so the seam stays where it was. + */ +export function pitchShift( + samples: Float32Array, + sampleRate: number, + semitones: number, + loop = false, +): Float32Array { + const length = samples.length; + const out = new Float32Array(length); + if (length === 0) return out; + const ratio = 2 ** (semitones / 12); + + const read = (index: number): number => { + if (loop) return samples[((index % length) + length) % length]!; + return index >= 0 && index < length ? samples[index]! : 0; + }; + const hop = Math.max(1, Math.round(HOP_SECONDS * sampleRate)); + const size = hop * 2; + const reach = Math.round(REACH_SECONDS * sampleRate); + // The pitched sound with room on either side for a frame that reaches past an end, so the loops + // below index it directly. + const pad = size + reach + COARSE; + const span = Math.max(1, Math.round(length / ratio)); + const pitched = new Float32Array(span + 2 * pad); + for (let i = -pad; i < span + pad; i++) { + if (!loop && (i < 0 || i >= span)) continue; + const position = (((i % span) + span) % span) * ratio; + const below = Math.floor(position); + const fraction = position - below; + pitched[i + pad] = read(below) * (1 - fraction) + read(below + 1) * fraction; + } + + const window = new Float32Array(size); + for (let j = 0; j < size; j++) window[j] = 0.5 - 0.5 * Math.cos((2 * Math.PI * j) / size); + const lowest = 0; + const highest = pitched.length - size; + const within = (from: number) => Math.max(lowest, Math.min(highest, from + pad)); + + let previous: number | null = null; + for (let start = -hop; start < length; start += hop) { + let from = within(Math.round((start + hop) / ratio) - hop); + if (previous !== null) from = bestFit(pitched, previous + hop, from, reach, hop, lowest, highest); + for (let j = Math.max(0, -start); j < size && start + j < length; j++) + out[start + j]! += window[j]! * pitched[from + j]!; + previous = from; + } + return out; +} + +/** + * Where near `around` the sound looks most like it does `hop` samples on from `follow`, searched + * coarsely and then finely, never outside `lowest`..`highest`. + */ +function bestFit( + sound: Float32Array, + follow: number, + around: number, + reach: number, + hop: number, + lowest: number, + highest: number, +): number { + const score = (from: number, stride: number): number => { + let sum = 0; + for (let j = 0; j < hop; j += stride) sum += sound[from + j]! * sound[follow + j]!; + return sum; + }; + const search = (centre: number, span: number, stride: number): number => { + let best = centre; + let bestScore = score(centre, stride); + for (let offset = -span; offset <= span; offset += stride) { + if (centre + offset < lowest || centre + offset > highest) continue; + const candidate = score(centre + offset, stride); + if (candidate > bestScore) { + best = centre + offset; + bestScore = candidate; + } + } + return best; + }; + return search(search(around, reach, COARSE), COARSE - 1, 1); +} diff --git a/src/viewport/SoundLibrary.ts b/src/viewport/SoundLibrary.ts index 307708bf..a7af8661 100644 --- a/src/viewport/SoundLibrary.ts +++ b/src/viewport/SoundLibrary.ts @@ -1,7 +1,10 @@ import { DRY, type FxChain, type FxStage } from '../core/soundfx/nodes'; +import { pitchShift } from '../core/soundfx/pitchShift'; import type { AudioSink } from '../runtime/systems/Audio'; const DEFAULT_BASE = '/game-maker/data/sounds'; +/** How many pitched copies are kept at once. */ +const SHIFTED_KEPT = 32; /** One sound in flight: its source, its own gain, the file it came from and who is making it. */ interface Voice { @@ -29,6 +32,8 @@ export class SoundLibrary implements AudioSink { private base = DEFAULT_BASE; private context: AudioContext | null = null; private readonly buffers = new Map(); + /** Pitched copies of decoded sounds, oldest use first (see `pitched`). */ + private readonly shifted = new Map(); private readonly loading = new Map>(); /** Never retried, so one broken reference cannot spam the network. */ private readonly failed = new Set(); @@ -170,12 +175,19 @@ export class SoundLibrary implements AudioSink { // voice would run silently and end as if it had played. if (context.state === 'suspended') void context.resume(); const source = context.createBufferSource(); - source.buffer = buffer; + const played = fx.pitch === 0 ? buffer : this.pitched(context, file, buffer, fx.pitch, loop); + source.buffer = played; source.loop = loop; - source.playbackRate.value = 2 ** (fx.pitch / 12); const gain = context.createGain(); gain.gain.value = Math.max(0, Math.min(1, volume)); - this.last = { file, volume: gain.gain.value, loop, pitch: fx.pitch, stages: fx.stages.length }; + this.last = { + file, + volume: gain.gain.value, + loop, + pitch: fx.pitch, + stages: fx.stages.length, + duration: played.duration, + }; let tail: AudioNode = source; for (const stage of fx.stages) tail = tail.connect(fxNode(context, stage)); tail.connect(gain).connect(context.destination); @@ -188,6 +200,35 @@ export class SoundLibrary implements AudioSink { source.start(); } + /** + * The sound at another pitch and its own length (#312), worked out once per file, pitch and loop + * and kept for the next voice. The oldest goes first, so a pitch drawn at random every play cannot + * fill the memory. + */ + private pitched( + context: AudioContext, + file: string, + buffer: AudioBuffer, + pitch: number, + loop: boolean, + ): AudioBuffer { + const key = `${file}\u0000${pitch}\u0000${loop}`; + const held = this.shifted.get(key); + if (held) { + this.shifted.delete(key); + this.shifted.set(key, held); + return held; + } + const shifted = context.createBuffer(buffer.numberOfChannels, buffer.length, buffer.sampleRate); + for (let channel = 0; channel < buffer.numberOfChannels; channel++) + shifted + .getChannelData(channel) + .set(pitchShift(buffer.getChannelData(channel), buffer.sampleRate, pitch, loop)); + this.shifted.set(key, shifted); + if (this.shifted.size > SHIFTED_KEPT) this.shifted.delete(this.shifted.keys().next().value!); + return shifted; + } + private audio(): AudioContext { return (this.context ??= new AudioContext()); } @@ -206,6 +247,8 @@ interface LastVoice { /** Semitones, and how many gain and filter nodes it passed. */ pitch: number; stages: number; + /** Seconds of what was played: a pitched voice is as long as the sound itself (#312). */ + duration: number; } function fxNode(context: AudioContext, stage: FxStage): AudioNode { diff --git a/tests/core/pitch-shift.test.ts b/tests/core/pitch-shift.test.ts new file mode 100644 index 00000000..482c1801 --- /dev/null +++ b/tests/core/pitch-shift.test.ts @@ -0,0 +1,56 @@ +/** + * The pitch of a sound FX (#312): a sound played higher or lower keeps its length. Measured on a + * sine, whose pitch is the number of times it rises through zero per second. + */ +import { describe, expect, it } from 'vitest'; +import { pitchShift } from '../../src/core/soundfx/pitchShift'; + +const RATE = 16000; + +function sine(frequency: number, seconds: number): Float32Array { + const samples = new Float32Array(Math.round(RATE * seconds)); + for (let i = 0; i < samples.length; i++) samples[i] = Math.sin((2 * Math.PI * frequency * i) / RATE); + return samples; +} + +/** Rises through zero per second over the middle half, away from the fade at either end. */ +function frequencyOf(samples: Float32Array): number { + const from = Math.floor(samples.length / 4); + const to = Math.floor((samples.length * 3) / 4); + let rises = 0; + for (let i = from + 1; i < to; i++) if (samples[i - 1]! < 0 && samples[i]! >= 0) rises++; + return rises / ((to - from) / RATE); +} + +const loudest = (samples: Float32Array) => samples.reduce((max, v) => Math.max(max, Math.abs(v)), 0); + +describe('pitch shifting', () => { + it.each([ + [12, 880], + [-12, 220], + [7, 440 * 2 ** (7 / 12)], + [0.5, 440 * 2 ** (0.5 / 12)], + [-30.5, 440 * 2 ** (-30.5 / 12)], + ])('moves a 440 Hz tone by %d semitones to about %d Hz, at the same length', (semitones, expected) => { + const tone = sine(440, 1); + const shifted = pitchShift(tone, RATE, semitones); + expect(shifted.length).toBe(tone.length); + expect(frequencyOf(shifted)).toBeGreaterThan(expected * 0.98); + expect(frequencyOf(shifted)).toBeLessThan(expected * 1.02); + }); + + it('leaves a sound alone at 0 and does not make it louder', () => { + const tone = sine(440, 0.5); + const same = pitchShift(tone, RATE, 0); + const middle = tone.length / 2; + expect(same[middle]).toBeCloseTo(tone[middle]!, 5); + expect(loudest(pitchShift(tone, RATE, 12))).toBeLessThanOrEqual(1.0001); + }); + + it('reads on across the end of a looping sound, so its seam does not fade out', () => { + const tone = sine(500, 0.25); + const tail = (samples: Float32Array) => loudest(samples.subarray(samples.length - 40)); + expect(tail(pitchShift(tone, RATE, 12))).toBeLessThan(0.6); + expect(tail(pitchShift(tone, RATE, 12, true))).toBeGreaterThan(0.95); + }); +}); diff --git a/tests/core/soundfx.test.ts b/tests/core/soundfx.test.ts index 3a59d9c1..bd045eee 100644 --- a/tests/core/soundfx.test.ts +++ b/tests/core/soundfx.test.ts @@ -237,10 +237,10 @@ describe('the parameters of a sound FX', () => { it('stay open in the plan and take the value handed in, held to the row', () => { const f = withParam('SoundFxPitch', 'semitones', [{ key: 'depth', kind: 'number', default: -12 }]); const plan = soundFxBuildFor(f.fx).compiled!; - expect(plan.pitch).toEqual([{ value: -12, param: 'depth', min: -24, max: 24 }]); + expect(plan.pitch).toEqual([{ value: -12, param: 'depth', min: -64, max: 64 }]); expect(fxChainOf(plan).pitch).toBe(-12); expect(fxChainOf(plan, { depth: 5 }).pitch).toBe(5); - expect(fxChainOf(plan, { depth: 100 }).pitch).toBe(24); + expect(fxChainOf(plan, { depth: 100 }).pitch).toBe(64); expect(fxChainOf(plan, { other: 3 }).pitch).toBe(-12); }); diff --git a/tests/viewport/sound-library.test.ts b/tests/viewport/sound-library.test.ts index c35cf6b9..e3b5c8ff 100644 --- a/tests/viewport/sound-library.test.ts +++ b/tests/viewport/sound-library.test.ts @@ -52,13 +52,26 @@ function wired( /** Resolves the pending decodes by hand, so a test decides when the bytes arrive. */ let decodes: (() => void)[] = []; +/** A mono buffer as Web Audio hands it out: a length, a rate and the samples. */ +function fakeBuffer(_channels: number, length: number, sampleRate: number) { + const data = new Float32Array(length); + return { + numberOfChannels: 1, + length, + sampleRate, + duration: length / sampleRate, + getChannelData: () => data, + }; +} + function fakeContext() { return { state: 'running', destination: { kind: 'destination' }, resume: () => Promise.resolve(), suspend: () => Promise.resolve(), - decodeAudioData: () => new Promise((resolve) => decodes.push(() => resolve({ duration: 1.5 }))), + decodeAudioData: () => new Promise((resolve) => decodes.push(() => resolve(fakeBuffer(1, 1200, 800)))), + createBuffer: fakeBuffer, createBufferSource: () => { const source: FakeSource = wired('source', { buffer: null, @@ -141,7 +154,14 @@ describe('the sound library', () => { await settle(); for (const resolve of decodes.splice(0)) resolve(); await settle(); - expect(library.stats().last).toEqual({ file: FILE, volume: 0.25, loop: true, pitch: 0, stages: 0 }); + expect(library.stats().last).toEqual({ + file: FILE, + volume: 0.25, + loop: true, + pitch: 0, + stages: 0, + duration: 1.5, + }); }); it('plays a dry voice straight into its volume', async () => { @@ -150,7 +170,7 @@ describe('the sound library', () => { expect(sources[0]?.playbackRate.value).toBe(1); }); - it('sends a voice through its sound FX in order, pitched at the source (#311)', async () => { + it('sends a voice through its sound FX in order (#311)', async () => { const library = new SoundLibrary(); const fx: FxChain = { pitch: 12, @@ -161,13 +181,26 @@ describe('the sound library', () => { }; await play(library, FILE, {}, fx); expect(wires).toEqual(['source > gain', 'gain > filter', 'filter > gain', 'gain > destination']); - expect(sources[0]?.playbackRate.value).toBe(2); expect(filters).toMatchObject([ { type: 'lowshelf', frequency: { value: 80 }, gain: { value: 6 }, Q: { value: 1 } }, ]); expect(library.stats().last).toMatchObject({ pitch: 12, stages: 2 }); }); + it('pitches a voice on a copy of its length rather than by speed, and keeps the copy (#312)', async () => { + const library = new SoundLibrary(); + await play(library); + await play(library, FILE, {}, { pitch: 7, stages: [] }); + await play(library, FILE, {}, { pitch: 7, stages: [] }); + await play(library, FILE, {}, { pitch: -3.5, stages: [] }); + const [dry, up, again, down] = sources.map((s) => s.buffer as { length: number; duration: number }); + expect(sources.map((s) => s.playbackRate.value)).toEqual([1, 1, 1, 1]); + expect(up).not.toBe(dry); + expect(again).toBe(up); + expect(down).not.toBe(up); + expect([up!.duration, down!.duration]).toEqual([dry!.duration, dry!.duration]); + }); + it('a stop while the bytes are in flight keeps the sound from starting afterwards', async () => { const library = new SoundLibrary(); await play(library, FILE, { hold: true }); diff --git a/tools/e2e/34-sound.mjs b/tools/e2e/34-sound.mjs index 03168487..400c329f 100644 --- a/tools/e2e/34-sound.mjs +++ b/tools/e2e/34-sound.mjs @@ -301,7 +301,7 @@ export async function sound(t) { const fxPalette = await json('__gm.overlayTexts()'); check( 'its palette offers the sound effects and no behavior node', - ['Input', 'Output', 'Gain', 'Pitch', 'EQ'].every((label) => fxPalette.includes(label)) && + ['Input', 'Output', 'Gain', 'Pitch', '5-Band-EQ'].every((label) => fxPalette.includes(label)) && !fxPalette.includes('Branch'), JSON.stringify(fxPalette.slice(0, 12)), ); @@ -336,10 +336,14 @@ export async function sound(t) { const fxPlayAt = await json('__gm.soundfx.playAt()'); await click(fxPlayAt.x, fxPlayAt.y); const heardFx = await waitFor(stats, (s) => s.started > beforeFx.started); + const blipSeconds = await json(`__gm.sound.duration(${JSON.stringify(blip.file)})`); check( - 'the tab’s Play sends the sound through the graph: an octave down and one gain stage', - heardFx.last?.file === blip.file && heardFx.last.pitch === -12 && heardFx.last.stages === 1, - JSON.stringify(heardFx), + 'the tab’s Play sends the sound through the graph: an octave down, as long as before, one gain stage', + heardFx.last?.file === blip.file && + heardFx.last.pitch === -12 && + heardFx.last.duration === blipSeconds && + heardFx.last.stages === 1, + JSON.stringify({ ...heardFx, blipSeconds }), ); await evaluate('__gm.sound.stopAll()'); await evaluate('__gm.soundfx.setParam("depth", 5)');