Pitch a sound FX without changing its length #312 #313 #314

The Pitch node changed the playback speed, so a sound pitched up was
also shorter and one pitched down longer. A pitch now keeps the
length, takes fractions and reaches further. Gain and the EQ bands
reach further down, and the EQ is renamed so a parametric one can
come later without a mix-up.

- Pitch shifting is the pure function pitchShift in
  core/soundfx/pitchShift.ts. It reads the sound faster or slower and
  stretches it back by WSOLA: 40 ms frames under a Hann window, each
  moved within 12.5 ms to where it continues the frame before it.
  A looping sound reads on across its end.
- SoundLibrary plays a pitched copy of the decoded buffer instead of
  setting playbackRate. Copies are kept per file, pitch and loop, the
  last 32 of them.
- stats().last tells the duration of what was played, so a check can
  see that the length is kept.
- Pitch: -64 to +64 semitones, scrubbed in steps of 0.1 (#312).
- Gain: -120 to +24 dB (#313).
- EQ is now labelled "5-Band-EQ", its bands go -60 to +24 dB (#314).
  The node type stays SoundFxEq, so saved graphs load unchanged.

Findings:
- A plain granular shifter was tried first and missed the pitch: at
  +0.5 semitones it measured 438 Hz instead of 452.9 Hz, because every
  grain restarts at the phase of the original. WSOLA measures 452 Hz.
- Far ends: +64 semitones from 440 Hz lands above half the sample rate
  and folds back, -64 is below what a 40 ms frame can follow. Nobody
  has listened to either.
- Speed: 3 s of a 48 kHz channel take 47 ms at +3 and +64, 171 ms
  at -64 (measured in Node).

Tests:
- pitch-shift (new): 7 tests. A 440 Hz sine at +12, -12, +7, +0.5 and
  -30.5 semitones is within 2 % of the target and keeps its length.
  Level stays at most 1, a loop does not fade at its seam.
- sound-library: 10 tests (+1), soundfx 22, node-catalog 368.
- Counter-check: with the pitched copy switched off the new
  sound-library test fails.
- e2e 34 (melektaus-atx): ALL PASSED - 28 checks. Its octave-down
  voice is now also checked for the length of the sound.
- Typecheck, ESLint and Prettier clean. The gate did not run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/core/soundfx/nodes.ts

@@ -123,7 +123,7 @@ export const SoundFxOutput: SoundFxNodeDef = {
   emit: (plan) => plan,
 };
 
-const GAIN = numberRow('gain', 'Gain (dB)', -60, 24, 0.5);
+const GAIN = numberRow('gain', 'Gain (dB)', -120, 24, 0.5);
 
 export const SoundFxGain: SoundFxNodeDef = {
   type: 'SoundFxGain',
@@ -140,14 +140,14 @@ export const SoundFxGain: SoundFxNodeDef = {
   },
 };
 
-const SEMITONES = numberRow('semitones', 'Semitones', -24, 24, 1);
+const SEMITONES = numberRow('semitones', 'Semitones', -64, 64, 0.1);
 
 export const SoundFxPitch: SoundFxNodeDef = {
   type: 'SoundFxPitch',
   label: 'Pitch',
   category: 'sound',
   description:
-    'Plays the sound higher or lower, in semitones: 12 is an octave up. It is the speed the sound is played at, so a higher one is also shorter, and it applies to the whole sound wherever the node sits in the line.',
+    'Plays the sound higher or lower, in semitones: 12 is an octave up, 0.5 a quarter tone. The sound keeps its length, and the pitch applies to the whole sound wherever the node sits in the line.',
   inputs: [audioIn, numberIn(SEMITONES)],
   outputs: [audioOut],
   params: [],
@@ -168,11 +168,11 @@ export const EQ_BANDS: readonly { key: string; label: string; filter: FxFilter;
 
 /** About an octave and a half wide: neighbouring peaks overlap, so no frequency falls between them. */
 const PEAK_Q = 1;
-const EQ_ROWS = EQ_BANDS.map((band) => numberRow(band.key, band.label, -24, 24, 0.5));
+const EQ_ROWS = EQ_BANDS.map((band) => numberRow(band.key, band.label, -60, 24, 0.5));
 
 export const SoundFxEq: SoundFxNodeDef = {
   type: 'SoundFxEq',
-  label: 'EQ',
+  label: '5-Band-EQ',
   category: 'sound',
   description:
     'A five-band equalizer: raises or lowers the lows (80 Hz), low mids (250 Hz), mids (1 kHz), high mids (4 kHz) and highs (12 kHz), each in decibels. A band at 0 does nothing.',

src/core/soundfx/pitchShift.ts

@@ -0,0 +1,95 @@
+/** Half a frame, in seconds: the hop between frames. A frame is longer than the period of a low note. */
+const HOP_SECONDS = 0.02;
+/** How far a frame may move to fit the one before it: a period of 80 Hz. */
+const REACH_SECONDS = 0.0125;
+/** The first search looks at every fourth offset and sample, the second at every one around the best. */
+const COARSE = 4;
+
+/**
+ * One channel of a sound played `semitones` higher or lower at its own length (#312). The sound is
+ * first read faster or slower, which sets the pitch and gets the length wrong, and then stretched
+ * back by WSOLA: frames half a frame apart are laid over each other under a Hann window, each one
+ * moved a little to where it continues the frame before it, so a tone carries on without a jump in
+ * phase. A sound that loops reads on across its end into its start, so the seam stays where it was.
+ */
+export function pitchShift(
+  samples: Float32Array,
+  sampleRate: number,
+  semitones: number,
+  loop = false,
+): Float32Array {
+  const length = samples.length;
+  const out = new Float32Array(length);
+  if (length === 0) return out;
+  const ratio = 2 ** (semitones / 12);
+
+  const read = (index: number): number => {
+    if (loop) return samples[((index % length) + length) % length]!;
+    return index >= 0 && index < length ? samples[index]! : 0;
+  };
+  const hop = Math.max(1, Math.round(HOP_SECONDS * sampleRate));
+  const size = hop * 2;
+  const reach = Math.round(REACH_SECONDS * sampleRate);
+  // The pitched sound with room on either side for a frame that reaches past an end, so the loops
+  // below index it directly.
+  const pad = size + reach + COARSE;
+  const span = Math.max(1, Math.round(length / ratio));
+  const pitched = new Float32Array(span + 2 * pad);
+  for (let i = -pad; i < span + pad; i++) {
+    if (!loop && (i < 0 || i >= span)) continue;
+    const position = (((i % span) + span) % span) * ratio;
+    const below = Math.floor(position);
+    const fraction = position - below;
+    pitched[i + pad] = read(below) * (1 - fraction) + read(below + 1) * fraction;
+  }
+
+  const window = new Float32Array(size);
+  for (let j = 0; j < size; j++) window[j] = 0.5 - 0.5 * Math.cos((2 * Math.PI * j) / size);
+  const lowest = 0;
+  const highest = pitched.length - size;
+  const within = (from: number) => Math.max(lowest, Math.min(highest, from + pad));
+
+  let previous: number | null = null;
+  for (let start = -hop; start < length; start += hop) {
+    let from = within(Math.round((start + hop) / ratio) - hop);
+    if (previous !== null) from = bestFit(pitched, previous + hop, from, reach, hop, lowest, highest);
+    for (let j = Math.max(0, -start); j < size && start + j < length; j++)
+      out[start + j]! += window[j]! * pitched[from + j]!;
+    previous = from;
+  }
+  return out;
+}
+
+/**
+ * Where near `around` the sound looks most like it does `hop` samples on from `follow`, searched
+ * coarsely and then finely, never outside `lowest`..`highest`.
+ */
+function bestFit(
+  sound: Float32Array,
+  follow: number,
+  around: number,
+  reach: number,
+  hop: number,
+  lowest: number,
+  highest: number,
+): number {
+  const score = (from: number, stride: number): number => {
+    let sum = 0;
+    for (let j = 0; j < hop; j += stride) sum += sound[from + j]! * sound[follow + j]!;
+    return sum;
+  };
+  const search = (centre: number, span: number, stride: number): number => {
+    let best = centre;
+    let bestScore = score(centre, stride);
+    for (let offset = -span; offset <= span; offset += stride) {
+      if (centre + offset < lowest || centre + offset > highest) continue;
+      const candidate = score(centre + offset, stride);
+      if (candidate > bestScore) {
+        best = centre + offset;
+        bestScore = candidate;
+      }
+    }
+    return best;
+  };
+  return search(search(around, reach, COARSE), COARSE - 1, 1);
+}

src/viewport/SoundLibrary.ts

@@ -1,7 +1,10 @@
 import { DRY, type FxChain, type FxStage } from '../core/soundfx/nodes';
+import { pitchShift } from '../core/soundfx/pitchShift';
 import type { AudioSink } from '../runtime/systems/Audio';
 
 const DEFAULT_BASE = '/game-maker/data/sounds';
+/** How many pitched copies are kept at once. */
+const SHIFTED_KEPT = 32;
 
 /** One sound in flight: its source, its own gain, the file it came from and who is making it. */
 interface Voice {
@@ -29,6 +32,8 @@ export class SoundLibrary implements AudioSink {
   private base = DEFAULT_BASE;
   private context: AudioContext | null = null;
   private readonly buffers = new Map<string, AudioBuffer>();
+  /** Pitched copies of decoded sounds, oldest use first (see `pitched`). */
+  private readonly shifted = new Map<string, AudioBuffer>();
   private readonly loading = new Map<string, Promise<AudioBuffer | null>>();
   /** Never retried, so one broken reference cannot spam the network. */
   private readonly failed = new Set<string>();
@@ -170,12 +175,19 @@ export class SoundLibrary implements AudioSink {
     // voice would run silently and end as if it had played.
     if (context.state === 'suspended') void context.resume();
     const source = context.createBufferSource();
-    source.buffer = buffer;
+    const played = fx.pitch === 0 ? buffer : this.pitched(context, file, buffer, fx.pitch, loop);
+    source.buffer = played;
     source.loop = loop;
-    source.playbackRate.value = 2 ** (fx.pitch / 12);
     const gain = context.createGain();
     gain.gain.value = Math.max(0, Math.min(1, volume));
-    this.last = { file, volume: gain.gain.value, loop, pitch: fx.pitch, stages: fx.stages.length };
+    this.last = {
+      file,
+      volume: gain.gain.value,
+      loop,
+      pitch: fx.pitch,
+      stages: fx.stages.length,
+      duration: played.duration,
+    };
     let tail: AudioNode = source;
     for (const stage of fx.stages) tail = tail.connect(fxNode(context, stage));
     tail.connect(gain).connect(context.destination);
@@ -188,6 +200,35 @@ export class SoundLibrary implements AudioSink {
     source.start();
   }
 
+  /**
+   * The sound at another pitch and its own length (#312), worked out once per file, pitch and loop
+   * and kept for the next voice. The oldest goes first, so a pitch drawn at random every play cannot
+   * fill the memory.
+   */
+  private pitched(
+    context: AudioContext,
+    file: string,
+    buffer: AudioBuffer,
+    pitch: number,
+    loop: boolean,
+  ): AudioBuffer {
+    const key = `${file}\u0000${pitch}\u0000${loop}`;
+    const held = this.shifted.get(key);
+    if (held) {
+      this.shifted.delete(key);
+      this.shifted.set(key, held);
+      return held;
+    }
+    const shifted = context.createBuffer(buffer.numberOfChannels, buffer.length, buffer.sampleRate);
+    for (let channel = 0; channel < buffer.numberOfChannels; channel++)
+      shifted
+        .getChannelData(channel)
+        .set(pitchShift(buffer.getChannelData(channel), buffer.sampleRate, pitch, loop));
+    this.shifted.set(key, shifted);
+    if (this.shifted.size > SHIFTED_KEPT) this.shifted.delete(this.shifted.keys().next().value!);
+    return shifted;
+  }
+
   private audio(): AudioContext {
     return (this.context ??= new AudioContext());
   }
@@ -206,6 +247,8 @@ interface LastVoice {
   /** Semitones, and how many gain and filter nodes it passed. */
   pitch: number;
   stages: number;
+  /** Seconds of what was played: a pitched voice is as long as the sound itself (#312). */
+  duration: number;
 }
 
 function fxNode(context: AudioContext, stage: FxStage): AudioNode {

tests/core/pitch-shift.test.ts

@@ -0,0 +1,56 @@
+/**
+ * The pitch of a sound FX (#312): a sound played higher or lower keeps its length. Measured on a
+ * sine, whose pitch is the number of times it rises through zero per second.
+ */
+import { describe, expect, it } from 'vitest';
+import { pitchShift } from '../../src/core/soundfx/pitchShift';
+
+const RATE = 16000;
+
+function sine(frequency: number, seconds: number): Float32Array {
+  const samples = new Float32Array(Math.round(RATE * seconds));
+  for (let i = 0; i < samples.length; i++) samples[i] = Math.sin((2 * Math.PI * frequency * i) / RATE);
+  return samples;
+}
+
+/** Rises through zero per second over the middle half, away from the fade at either end. */
+function frequencyOf(samples: Float32Array): number {
+  const from = Math.floor(samples.length / 4);
+  const to = Math.floor((samples.length * 3) / 4);
+  let rises = 0;
+  for (let i = from + 1; i < to; i++) if (samples[i - 1]! < 0 && samples[i]! >= 0) rises++;
+  return rises / ((to - from) / RATE);
+}
+
+const loudest = (samples: Float32Array) => samples.reduce((max, v) => Math.max(max, Math.abs(v)), 0);
+
+describe('pitch shifting', () => {
+  it.each([
+    [12, 880],
+    [-12, 220],
+    [7, 440 * 2 ** (7 / 12)],
+    [0.5, 440 * 2 ** (0.5 / 12)],
+    [-30.5, 440 * 2 ** (-30.5 / 12)],
+  ])('moves a 440 Hz tone by %d semitones to about %d Hz, at the same length', (semitones, expected) => {
+    const tone = sine(440, 1);
+    const shifted = pitchShift(tone, RATE, semitones);
+    expect(shifted.length).toBe(tone.length);
+    expect(frequencyOf(shifted)).toBeGreaterThan(expected * 0.98);
+    expect(frequencyOf(shifted)).toBeLessThan(expected * 1.02);
+  });
+
+  it('leaves a sound alone at 0 and does not make it louder', () => {
+    const tone = sine(440, 0.5);
+    const same = pitchShift(tone, RATE, 0);
+    const middle = tone.length / 2;
+    expect(same[middle]).toBeCloseTo(tone[middle]!, 5);
+    expect(loudest(pitchShift(tone, RATE, 12))).toBeLessThanOrEqual(1.0001);
+  });
+
+  it('reads on across the end of a looping sound, so its seam does not fade out', () => {
+    const tone = sine(500, 0.25);
+    const tail = (samples: Float32Array) => loudest(samples.subarray(samples.length - 40));
+    expect(tail(pitchShift(tone, RATE, 12))).toBeLessThan(0.6);
+    expect(tail(pitchShift(tone, RATE, 12, true))).toBeGreaterThan(0.95);
+  });
+});

tests/core/soundfx.test.ts

@@ -237,10 +237,10 @@ describe('the parameters of a sound FX', () => {
   it('stay open in the plan and take the value handed in, held to the row', () => {
     const f = withParam('SoundFxPitch', 'semitones', [{ key: 'depth', kind: 'number', default: -12 }]);
     const plan = soundFxBuildFor(f.fx).compiled!;
-    expect(plan.pitch).toEqual([{ value: -12, param: 'depth', min: -24, max: 24 }]);
+    expect(plan.pitch).toEqual([{ value: -12, param: 'depth', min: -64, max: 64 }]);
     expect(fxChainOf(plan).pitch).toBe(-12);
     expect(fxChainOf(plan, { depth: 5 }).pitch).toBe(5);
-    expect(fxChainOf(plan, { depth: 100 }).pitch).toBe(24);
+    expect(fxChainOf(plan, { depth: 100 }).pitch).toBe(64);
     expect(fxChainOf(plan, { other: 3 }).pitch).toBe(-12);
   });
 

tests/viewport/sound-library.test.ts

@@ -52,13 +52,26 @@ function wired<T extends object>(
 /** Resolves the pending decodes by hand, so a test decides when the bytes arrive. */
 let decodes: (() => void)[] = [];
 
+/** A mono buffer as Web Audio hands it out: a length, a rate and the samples. */
+function fakeBuffer(_channels: number, length: number, sampleRate: number) {
+  const data = new Float32Array(length);
+  return {
+    numberOfChannels: 1,
+    length,
+    sampleRate,
+    duration: length / sampleRate,
+    getChannelData: () => data,
+  };
+}
+
 function fakeContext() {
   return {
     state: 'running',
     destination: { kind: 'destination' },
     resume: () => Promise.resolve(),
     suspend: () => Promise.resolve(),
-    decodeAudioData: () => new Promise((resolve) => decodes.push(() => resolve({ duration: 1.5 }))),
+    decodeAudioData: () => new Promise((resolve) => decodes.push(() => resolve(fakeBuffer(1, 1200, 800)))),
+    createBuffer: fakeBuffer,
     createBufferSource: () => {
       const source: FakeSource = wired('source', {
         buffer: null,
@@ -141,7 +154,14 @@ describe('the sound library', () => {
     await settle();
     for (const resolve of decodes.splice(0)) resolve();
     await settle();
-    expect(library.stats().last).toEqual({ file: FILE, volume: 0.25, loop: true, pitch: 0, stages: 0 });
+    expect(library.stats().last).toEqual({
+      file: FILE,
+      volume: 0.25,
+      loop: true,
+      pitch: 0,
+      stages: 0,
+      duration: 1.5,
+    });
   });
 
   it('plays a dry voice straight into its volume', async () => {
@@ -150,7 +170,7 @@ describe('the sound library', () => {
     expect(sources[0]?.playbackRate.value).toBe(1);
   });
 
-  it('sends a voice through its sound FX in order, pitched at the source (#311)', async () => {
+  it('sends a voice through its sound FX in order (#311)', async () => {
     const library = new SoundLibrary();
     const fx: FxChain = {
       pitch: 12,
@@ -161,13 +181,26 @@ describe('the sound library', () => {
     };
     await play(library, FILE, {}, fx);
     expect(wires).toEqual(['source > gain', 'gain > filter', 'filter > gain', 'gain > destination']);
-    expect(sources[0]?.playbackRate.value).toBe(2);
     expect(filters).toMatchObject([
       { type: 'lowshelf', frequency: { value: 80 }, gain: { value: 6 }, Q: { value: 1 } },
     ]);
     expect(library.stats().last).toMatchObject({ pitch: 12, stages: 2 });
   });
 
+  it('pitches a voice on a copy of its length rather than by speed, and keeps the copy (#312)', async () => {
+    const library = new SoundLibrary();
+    await play(library);
+    await play(library, FILE, {}, { pitch: 7, stages: [] });
+    await play(library, FILE, {}, { pitch: 7, stages: [] });
+    await play(library, FILE, {}, { pitch: -3.5, stages: [] });
+    const [dry, up, again, down] = sources.map((s) => s.buffer as { length: number; duration: number });
+    expect(sources.map((s) => s.playbackRate.value)).toEqual([1, 1, 1, 1]);
+    expect(up).not.toBe(dry);
+    expect(again).toBe(up);
+    expect(down).not.toBe(up);
+    expect([up!.duration, down!.duration]).toEqual([dry!.duration, dry!.duration]);
+  });
+
   it('a stop while the bytes are in flight keeps the sound from starting afterwards', async () => {
     const library = new SoundLibrary();
     await play(library, FILE, { hold: true });

tools/e2e/34-sound.mjs

@@ -301,7 +301,7 @@ export async function sound(t) {
   const fxPalette = await json('__gm.overlayTexts()');
   check(
     'its palette offers the sound effects and no behavior node',
-    ['Input', 'Output', 'Gain', 'Pitch', 'EQ'].every((label) => fxPalette.includes(label)) &&
+    ['Input', 'Output', 'Gain', 'Pitch', '5-Band-EQ'].every((label) => fxPalette.includes(label)) &&
       !fxPalette.includes('Branch'),
     JSON.stringify(fxPalette.slice(0, 12)),
   );
@@ -336,10 +336,14 @@ export async function sound(t) {
   const fxPlayAt = await json('__gm.soundfx.playAt()');
   await click(fxPlayAt.x, fxPlayAt.y);
   const heardFx = await waitFor(stats, (s) => s.started > beforeFx.started);
+  const blipSeconds = await json(`__gm.sound.duration(${JSON.stringify(blip.file)})`);
   check(
-    'the tab’s Play sends the sound through the graph: an octave down and one gain stage',
+    'the tab’s Play sends the sound through the graph: an octave down, as long as before, one gain stage',
-    heardFx.last?.file === blip.file && heardFx.last.pitch === -12 && heardFx.last.stages === 1,
+    heardFx.last?.file === blip.file &&
-    JSON.stringify(heardFx),
+      heardFx.last.pitch === -12 &&
+      heardFx.last.duration === blipSeconds &&
+      heardFx.last.stages === 1,
+    JSON.stringify({ ...heardFx, blipSeconds }),
   );
   await evaluate('__gm.sound.stopAll()');
   await evaluate('__gm.soundfx.setParam("depth", 5)');