Play a sound through a graph of gain, pitch and EQ effects #311

A sound could only be played as it was uploaded. A Sound FX asset is
a node graph like a shader: Input, effects, Output. Play sound has a
new input fx that sends the voice through it.

- New asset type soundfx ("Sound FX"), not placeable, with a graph
  and its own tab. The heading in the asset panel is "Sound effects",
  so it does not take the click meant for the + menu entry.
- Nodes in core/soundfx/nodes.ts, category sound, port type audio
  that only fits audio: Input, Output, Gain (dB), Pitch (semitones),
  EQ (80 Hz and 12 kHz shelves, peaks at 250 Hz, 1 kHz and 4 kHz).
- compileSoundFx walks back from the Output with the shared graph
  emitter. An input takes one edge, so the result is one line: a
  pitch and a list of gain and filter stages. A node without input
  or a graph without Output reports a problem on the node.
- Pitch is the playback rate of the source. It applies to the whole
  sound and also changes its length, the node description says so.
- The BehaviorSystem resolves fx and hands the line to the AudioSink.
  An fx that is missing or does not compile plays the sound dry.
- SoundLibrary wires source, stages, volume, destination in that
  order. stats().last now also names pitch and stage count.
- The tab header picks one of the project's sounds and plays it
  through the graph as it stands. Stop there ends only its own voice.

Tests:
- soundfx.test.ts (new, 11), sound-library (9, +2 for the wiring),
  node-catalog (+1 for fx), graph and asset-groups adjusted
- e2e 34: ALL PASSED - 26 checks (+5: + menu opens the tab, palette,
  2 effects, tab Play and game Play through the FX at -12 st)
- Gate on melektaus-atx: typecheck, lint, prettier clean, 239 of 240
  test files green. The red one was sound-library (its fake
  AudioContext had no playbackRate), fixed afterwards, gate not
  re-run in full.
- Counter-check: without the audio rule in portsCompatible the port
  test fails.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/graph.test.ts

@@ -79,8 +79,10 @@ describe('node registry', () => {
       if (def.category === 'event') expect(def.inputs.filter((p) => p.type === 'exec')).toHaveLength(0);
       // A state is a name in a machine; its ports only anchor transitions, so it never runs.
       else if (def.category === 'state') expect(def.run).toBeUndefined();
-      // A shader node is compiled to GLSL and a particle node to a function: neither is ever run.
-      else if (def.category === 'shader' || def.category === 'particles') expect('emit' in def).toBe(true);
+      // A shader node is compiled to GLSL, a particle node to a function and a sound FX node to a
+      // stage of a voice: none is ever run.
+      else if (def.category === 'shader' || def.category === 'particles' || def.category === 'sound')
+        expect('emit' in def).toBe(true);
       // The end of a block runs as part of the node that opens it (`Find` … `Match`).
       else if (defs.some((d) => d.opens?.type === def.type)) expect(def.run).toBeUndefined();
       // Everything else does one of the two: a step (`run`), or an answer whenever asked (`read`).

tests/core/node-catalog.test.ts

@@ -1950,7 +1950,7 @@ const CASES: Record<string, Case[]> = {
       // flag – is the BehaviorSystem's to resolve, since only it holds the project (see
       // `sounds.test.ts`).
       after: (ctx) =>
-        expect(ctx.played).toEqual([{ assetId: 'snd-1', volume: 0.5, loop: true, detached: false }]),
+        expect(ctx.played).toEqual([{ assetId: 'snd-1', volume: 0.5, loop: true, detached: false, fx: '' }]),
     },
     {
       name: 'detached says the voice is nobody’s, so destroying this object leaves it playing',
@@ -1970,6 +1970,12 @@ const CASES: Record<string, Case[]> = {
       next: 'exec',
       after: (ctx) => expect(ctx.played[0]?.loop).toBe(false),
     },
+    {
+      name: 'hands on the sound FX it names (#311)',
+      inputs: { sound: 'snd-1', volume: 1, loop: '', fx: 'fx-1' },
+      next: 'exec',
+      after: (ctx) => expect(ctx.played[0]?.fx).toBe('fx-1'),
+    },
   ],
   LoadScene: [
     {

tests/core/soundfx.test.ts

@@ -0,0 +1,219 @@
+/**
+ * A sound FX asset (#311) is a graph a sound is played through: Input, effects, Output. It never
+ * runs as a behavior does – the compiler turns it into the line of effects a voice passes.
+ */
+import { describe, expect, it } from 'vitest';
+import { buildAsset, CreateAssetCommand, CreateSoundCommand } from '../../src/core/commands/assetCommands';
+import { portsCompatible } from '../../src/core/graph/Graph';
+import type { BehaviorAsset, SoundFxAsset } from '../../src/core/model/Asset';
+import { isPlaceableType } from '../../src/core/model/AssetTypes';
+import { assetRefChoices } from '../../src/core/properties/assetRef';
+import { compileSoundFx, soundFxBuildFor } from '../../src/core/soundfx/compile';
+import { DRY, type FxChain } from '../../src/core/soundfx/nodes';
+import { deserializeProject } from '../../src/core/store/serialize';
+import type { AudioSink } from '../../src/runtime/systems/Audio';
+import { paletteEntries } from '../../src/editor/graph/paletteEntries';
+import {
+  graphEditor,
+  makeBehavior,
+  makeProject,
+  makeSpriteScene,
+  place,
+  type ProjectFixture,
+  roundTrip,
+  startWorld,
+} from '../helpers';
+
+const FILE = `${'a1'.repeat(32)}.wav`;
+
+function makeFx(f: ProjectFixture = makeProject()) {
+  const command = new CreateAssetCommand('soundfx', 'Cave');
+  f.stack.execute(command);
+  const fx = command.asset as SoundFxAsset;
+  return { ...f, fx, edit: graphEditor(f, fx as unknown as BehaviorAsset) };
+}
+
+/** Input, the given effects in a row, Output. */
+function line(effects: { type: string; params?: Record<string, number> }[]) {
+  const f = makeFx();
+  const input = f.edit.addNode('SoundFxInput');
+  let tail = input;
+  for (const effect of effects) {
+    const node = f.edit.addNode(effect.type, effect.params ?? {});
+    f.edit.connect(tail, 'out', node, 'in');
+    tail = node;
+  }
+  const output = f.edit.addNode('SoundFxOutput');
+  f.edit.connect(tail, 'out', output, 'in');
+  return { ...f, input, output };
+}
+
+describe('the sound FX asset', () => {
+  it('is not placeable, starts with an empty graph and is what the fx picker offers', () => {
+    expect(isPlaceableType('soundfx')).toBe(false);
+    expect(buildAsset('soundfx', 'FX')).toMatchObject({ type: 'soundfx', graph: { nodes: [], edges: [] } });
+    const f = makeFx();
+    f.stack.execute(new CreateSoundCommand('Jump', FILE));
+    expect(assetRefChoices(f.project, ['soundfx']).map((a) => a.id)).toEqual([f.fx.id]);
+  });
+
+  it('survives a round trip and keeps only sound FX nodes', () => {
+    const f = line([{ type: 'SoundFxGain', params: { gain: -6 } }]);
+    expect(roundTrip(f.project)).toEqual(f.project);
+
+    const loaded = deserializeProject({
+      name: 'P',
+      assets: [
+        {
+          id: 'fx',
+          type: 'soundfx',
+          name: 'FX',
+          graph: {
+            nodes: [
+              { id: 'a', type: 'SoundFxInput', x: 0, y: 0, params: {} },
+              { id: 'b', type: 'OnStart', x: 0, y: 0, params: {} },
+            ],
+            edges: [],
+          },
+        },
+      ],
+    });
+    expect((loaded.assets[0] as SoundFxAsset).graph.nodes.map((n) => n.id)).toEqual(['a']);
+  });
+
+  it('carries sound only from a sound port into a sound port', () => {
+    expect(portsCompatible('audio', 'audio')).toBe(true);
+    expect(portsCompatible('audio', 'any')).toBe(false);
+    expect(portsCompatible('number', 'audio')).toBe(false);
+    expect(portsCompatible('any', 'audio')).toBe(false);
+  });
+
+  it('offers its effects, Input and Output once each, and none of them in a behavior', () => {
+    const f = makeFx();
+    const ids = () => paletteEntries(f.project, f.fx).map((e) => e.id);
+    expect(ids()).toEqual(['SoundFxInput', 'SoundFxOutput', 'SoundFxGain', 'SoundFxPitch', 'SoundFxEq']);
+    f.edit.addNode('SoundFxInput');
+    f.edit.addNode('SoundFxOutput');
+    expect(ids()).toEqual(['SoundFxGain', 'SoundFxPitch', 'SoundFxEq']);
+
+    const behavior = makeBehavior(f, 'Logic');
+    expect(paletteEntries(f.project, behavior).some((e) => e.def.category === 'sound')).toBe(false);
+  });
+});
+
+describe('compiling a sound FX graph', () => {
+  it('needs an Output, and an Output fed from the Input', () => {
+    const f = makeFx();
+    expect(compileSoundFx(f.fx.graph)).toMatchObject({ compiled: null, reason: 'No Output node' });
+
+    const output = f.edit.addNode('SoundFxOutput');
+    const gain = f.edit.addNode('SoundFxGain');
+    f.edit.connect(gain, 'out', output, 'in');
+    const build = compileSoundFx(f.fx.graph);
+    expect(build.compiled).toBeNull();
+    expect(build.problems[gain.id]).toBe('Nothing comes in: connect it to the Input');
+  });
+
+  it('passes the sound on untouched through Input and Output alone', () => {
+    const build = compileSoundFx(line([]).fx.graph);
+    expect(build).toMatchObject({ compiled: DRY, effects: 0, reason: '' });
+  });
+
+  it('turns every node on the line into its stages, in order', () => {
+    const f = line([
+      { type: 'SoundFxGain', params: { gain: -6 } },
+      { type: 'SoundFxPitch', params: { semitones: 12 } },
+      { type: 'SoundFxEq', params: { low: 6, high: -3 } },
+      { type: 'SoundFxPitch', params: { semitones: -5 } },
+    ]);
+    const build = compileSoundFx(f.fx.graph);
+    expect(build.effects).toBe(4);
+    const chain = build.compiled!;
+    expect(chain.pitch).toBe(7);
+    expect(chain.stages).toHaveLength(3);
+    expect(chain.stages[0]!.kind).toBe('gain');
+    expect(chain.stages[0]!.gain).toBeCloseTo(0.501, 3);
+    expect(chain.stages.slice(1)).toEqual([
+      { kind: 'filter', filter: 'lowshelf', frequency: 80, gain: 6, q: 1 },
+      { kind: 'filter', filter: 'highshelf', frequency: 12000, gain: -3, q: 1 },
+    ]);
+  });
+
+  it('leaves out what does nothing and holds a value to its row', () => {
+    const quiet = compileSoundFx(line([{ type: 'SoundFxGain' }, { type: 'SoundFxEq' }]).fx.graph);
+    expect(quiet.compiled).toEqual(DRY);
+    expect(quiet.effects).toBe(2);
+
+    const loud = compileSoundFx(line([{ type: 'SoundFxGain', params: { gain: 200 } }]).fx.graph).compiled!;
+    expect(loud.stages[0]!.gain).toBeCloseTo(10 ** (24 / 20));
+  });
+
+  it('ignores a node hanging off the line', () => {
+    const f = line([{ type: 'SoundFxPitch', params: { semitones: 3 } }]);
+    const aside = f.edit.addNode('SoundFxGain', { gain: -20 });
+    f.edit.connect(f.input, 'out', aside, 'in');
+    const build = compileSoundFx(f.fx.graph);
+    expect(build.compiled).toEqual({ pitch: 3, stages: [] });
+    expect(build.effects).toBe(1);
+  });
+});
+
+/** Records what it was asked to play, with the line of effects. */
+function fakeAudio(): AudioSink & { played: { file: string; fx: FxChain | undefined }[] } {
+  const sink = {
+    played: [] as { file: string; fx: FxChain | undefined }[],
+    play: (file: string, _volume: number, _loop: boolean, _owner: string, fx?: FxChain) =>
+      sink.played.push({ file, fx }),
+    stop: () => {},
+    stopOwner: () => {},
+  };
+  return sink;
+}
+
+describe('Play sound with fx', () => {
+  function playingScene(fxId: (f: ReturnType<typeof makeFx>) => string) {
+    const scene = makeSpriteScene('Box');
+    const withFx = makeFx(scene);
+    const fx = withFx.fx;
+    const input = withFx.edit.addNode('SoundFxInput');
+    const pitch = withFx.edit.addNode('SoundFxPitch', { semitones: -12 });
+    const output = withFx.edit.addNode('SoundFxOutput');
+    withFx.edit.connect(input, 'out', pitch, 'in');
+    withFx.edit.connect(pitch, 'out', output, 'in');
+    const sound = new CreateSoundCommand('Jump', FILE);
+    scene.stack.execute(sound);
+    const behavior = makeBehavior(scene, 'Logic');
+    place(scene, behavior.id, { x: 0, y: 0 }, place(scene, scene.spriteId, { x: 0, y: 0 }));
+    const edit = graphEditor(scene, behavior);
+    const start = edit.addNode('OnStart');
+    const play = edit.addNode('PlaySound', { sound: sound.asset.id, fx: fxId(withFx) });
+    edit.connect(start, 'exec', play, 'exec');
+    return { ...scene, fx, output, pitch };
+  }
+
+  it('sends the voice through the asset the node names', () => {
+    const f = playingScene((s) => s.fx.id);
+    const audio = fakeAudio();
+    startWorld(f, { behaviors: true, audio }).frame();
+    expect(audio.played).toEqual([{ file: FILE, fx: { pitch: -12, stages: [] } }]);
+  });
+
+  it('plays it dry without an fx, with one that is gone, and with one that does not compile', () => {
+    const none = fakeAudio();
+    startWorld(
+      playingScene(() => ''),
+      { behaviors: true, audio: none },
+    ).frame();
+    const gone = fakeAudio();
+    startWorld(
+      playingScene(() => 'no-such-fx'),
+      { behaviors: true, audio: gone },
+    ).frame();
+    const broken = playingScene((s) => s.fx.id);
+    broken.fx.graph.edges = [];
+    expect(soundFxBuildFor(broken.fx).compiled).toBeNull();
+    const brokenAudio = fakeAudio();
+    startWorld(broken, { behaviors: true, audio: brokenAudio }).frame();
+    for (const audio of [none, gone, brokenAudio]) expect(audio.played).toEqual([{ file: FILE, fx: DRY }]);
+  });
+});

tests/helpers.ts

@@ -205,7 +205,7 @@ export type FakeContext = NodeContext & {
   stopped: boolean;
   changes: { machineId: string; from: string; to: string }[];
   /** What the context was asked to play and to silence, in order – nothing here makes a noise. */
-  played: { assetId: string; volume: number; loop: boolean | undefined; detached: boolean }[];
+  played: { assetId: string; volume: number; loop: boolean | undefined; detached: boolean; fx: string }[];
   silenced: string[];
   /** The scenes the context was asked to load, in order. */
   loaded: string[];
@@ -258,7 +258,8 @@ export function fakeContext(overrides: Partial<NodeContext> = {}): FakeContext {
     played: [],
     silenced: [],
     loaded: [],
-    playSound: (assetId, volume, loop, detached) => ctx.played.push({ assetId, volume, loop, detached }),
+    playSound: (assetId, volume, loop, detached, fx) =>
+      ctx.played.push({ assetId, volume, loop, detached, fx }),
     stopSound: (assetId) => ctx.silenced.push(assetId),
     loadScene: (sceneId) => ctx.loaded.push(sceneId),
     spawn: () => null,

tests/viewport/sound-library.test.ts

@@ -6,6 +6,7 @@
  * The device is stubbed – nothing makes a noise in a unit test, and Web Audio does not exist here.
  */
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import type { FxChain } from '../../src/core/soundfx/nodes';
 import { SoundLibrary } from '../../src/viewport/SoundLibrary';
 
 const FILE = 'a.wav';
@@ -14,6 +15,7 @@ const OTHER = 'b.wav';
 interface FakeSource {
   buffer: unknown;
   loop: boolean;
+  playbackRate: { value: number };
   onended: (() => void) | null;
   started: boolean;
   stopped: boolean;
@@ -23,6 +25,29 @@ interface FakeSource {
 }
 
 const sources: FakeSource[] = [];
+/** Every `connect` made, as "from > to" by the kind of node – the order a voice is wired in. */
+const wires: string[] = [];
+const filters: {
+  type: string;
+  frequency: { value: number };
+  gain: { value: number };
+  Q: { value: number };
+}[] = [];
+
+/** A node that logs where it is connected to and hands the target back, as Web Audio's does. */
+function wired<T extends object>(
+  kind: string,
+  node: T,
+): T & { kind: string; connect: (to: unknown) => unknown } {
+  return {
+    ...node,
+    kind,
+    connect: (to: unknown) => {
+      wires.push(`${kind} > ${(to as { kind?: string }).kind ?? '?'}`);
+      return to;
+    },
+  };
+}
 
 /** Resolves the pending decodes by hand, so a test decides when the bytes arrive. */
 let decodes: (() => void)[] = [];
@@ -30,29 +55,39 @@ let decodes: (() => void)[] = [];
 function fakeContext() {
   return {
     state: 'running',
-    destination: {},
+    destination: { kind: 'destination' },
     resume: () => Promise.resolve(),
     suspend: () => Promise.resolve(),
     decodeAudioData: () => new Promise((resolve) => decodes.push(() => resolve({ duration: 1.5 }))),
     createBufferSource: () => {
-      const source: FakeSource = {
+      const source: FakeSource = wired('source', {
         buffer: null,
         loop: false,
+        playbackRate: { value: 1 },
         onended: null,
         started: false,
         stopped: false,
-        connect: (target: unknown) => target,
         start: () => {
           source.started = true;
         },
         stop: () => {
           source.stopped = true;
         },
-      };
+      });
       sources.push(source);
       return source;
     },
-    createGain: () => ({ gain: { value: 1 }, connect: (target: unknown) => target }),
+    createGain: () => wired('gain', { gain: { value: 1 } }),
+    createBiquadFilter: () => {
+      const filter = wired('filter', {
+        type: '',
+        frequency: { value: 0 },
+        gain: { value: 0 },
+        Q: { value: 0 },
+      });
+      filters.push(filter);
+      return filter;
+    },
   };
 }
 
@@ -61,6 +96,8 @@ const settle = () => new Promise((resolve) => setTimeout(resolve, 0));
 
 beforeEach(() => {
   sources.length = 0;
+  wires.length = 0;
+  filters.length = 0;
   decodes = [];
   vi.stubGlobal(
     'AudioContext',
@@ -80,8 +117,8 @@ afterEach(() => {
 });
 
 /** Plays `file` and lets its bytes arrive, unless `hold` keeps the decode pending. */
-async function play(library: SoundLibrary, file = FILE, { hold = false } = {}): Promise<void> {
-  library.play(file, 1, false);
+async function play(library: SoundLibrary, file = FILE, { hold = false } = {}, fx?: FxChain): Promise<void> {
+  library.play(file, 1, false, '', fx);
   await settle();
   if (hold) return;
   for (const resolve of decodes.splice(0)) resolve();
@@ -104,7 +141,31 @@ describe('the sound library', () => {
     await settle();
     for (const resolve of decodes.splice(0)) resolve();
     await settle();
-    expect(library.stats().last).toEqual({ file: FILE, volume: 0.25, loop: true });
+    expect(library.stats().last).toEqual({ file: FILE, volume: 0.25, loop: true, pitch: 0, stages: 0 });
+  });
+
+  it('plays a dry voice straight into its volume', async () => {
+    await play(new SoundLibrary());
+    expect(wires).toEqual(['source > gain', 'gain > destination']);
+    expect(sources[0]?.playbackRate.value).toBe(1);
+  });
+
+  it('sends a voice through its sound FX in order, pitched at the source (#311)', async () => {
+    const library = new SoundLibrary();
+    const fx: FxChain = {
+      pitch: 12,
+      stages: [
+        { kind: 'gain', gain: 0.5 },
+        { kind: 'filter', filter: 'lowshelf', frequency: 80, gain: 6, q: 1 },
+      ],
+    };
+    await play(library, FILE, {}, fx);
+    expect(wires).toEqual(['source > gain', 'gain > filter', 'filter > gain', 'gain > destination']);
+    expect(sources[0]?.playbackRate.value).toBe(2);
+    expect(filters).toMatchObject([
+      { type: 'lowshelf', frequency: { value: 80 }, gain: { value: 6 }, Q: { value: 1 } },
+    ]);
+    expect(library.stats().last).toMatchObject({ pitch: 12, stages: 2 });
   });
 
   it('a stop while the bytes are in flight keeps the sound from starting afterwards', async () => {