Blend two ways of a sound FX with a Mix node #316

A sound FX was a single line from Input to Output, so two versions
of one sound could not be heard together. The new Mix node joins two
ways again, and each way can carry its own effects, its own Pitch
included.

- Mix has the audio inputs a and b and a number input mix (0 to 1,
  default 0.5). At 0 only a is heard, at 1 only b. Like every number
  it can come from a parameter of the Input and is held to its range.
- The compiled plan and the chain a voice gets are now a tree. A
  source carries a pitch and stages, a mix carries a, b and the
  stages after it. A Mix that ends at exactly 0 or 1 collapses to the
  one way heard, so the silent way is never worked out.
- A Pitch is still worked out on the sound as it starts (#312). Behind
  a Mix it goes to both of its sounds. An effect before a Pitch
  therefore acts as if it came after it, the node now says so.
- The player starts one source per pitch, all at once and all of the
  same length, and two ways at the same pitch share one. A mix becomes
  two gain nodes with 1 - mix and mix into a sum. A voice leaves the
  count when all of its sources have ended.
- stats().last reports pitches (one per source) instead of pitch.

Tests:
- soundfx.test.ts: 27 tests (+5), sound-library.test.ts: 12 (+2),
  node-catalog.test.ts: 368, all green. Typecheck, ESLint, Prettier
  clean.
- Counter-check: a Pitch behind the Mix given to a only, and b mixed
  at 1 - mix, each fails its new test.
- e2e 34 on melektaus-atx: ALL PASSED, 29 checks (+1). The new check
  plays the pitched way mixed with the dry Input in the browser: two
  sources at 5 and 0 semitones, as long as the sound itself.
- The full gate did not run before this commit.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/core/soundfx/compile.ts

@@ -41,10 +41,10 @@ function inputOutputs(node: GraphNode): Record<string, FxValue> {
 }
 
 /**
- * A sound FX graph to the line a voice is sent along: walked back from the Output to the Input, every
- * node on the way adding what it does. An input takes one edge, so the line never forks on its way
- * to the Output; a node fed from nowhere is a problem rather than silence. A number fed from a
- * parameter stays open in the plan until a voice is started (#311).
+ * A sound FX graph to the way a voice is sent: walked back from the Output to the Input, every node
+ * on the way adding what it does. An input takes one edge, so only a Mix joins two ways into one
+ * (#316); a node fed from nowhere is a problem rather than silence. A number fed from a parameter
+ * stays open in the plan until a voice is started (#311).
  */
 export function compileSoundFx(graph: Graph): SoundFxBuild {
   const walk = graphEmitter<SoundFxNodeDef, FxValue>(graph, {
@@ -53,10 +53,11 @@ export function compileSoundFx(graph: Graph): SoundFxBuild {
     emit: (def, inputs, node) => {
       if (node.type === SOUND_FX_INPUT) return inputOutputs(node);
       const amounts: Record<string, FxAmount> = {};
-      for (const [key, value] of Object.entries(inputs)) if ('amount' in value) amounts[key] = value.amount;
-      const arriving = inputs.in;
-      const plan = arriving && 'plan' in arriving ? arriving.plan : EMPTY_PLAN;
-      return { out: { plan: def.emit(plan, amounts) } };
+      const sounds: Record<string, FxPlan> = {};
+      for (const [key, value] of Object.entries(inputs))
+        if ('amount' in value) amounts[key] = value.amount;
+        else sounds[key] = value.plan;
+      return { out: { plan: def.emit(sounds, amounts) } };
     },
     unfed: (node, port) => {
       if (port.type === 'number') return { amount: { value: asNumber(inlineValue(node, port)) } };

src/core/soundfx/nodes.ts

@@ -15,12 +15,12 @@ export type FxStage =
 
 /**
  * A sound on its way through the graph: how many semitones it is played up or down, and what it
- * passes, in order. `gain` is a factor, a filter's `gain` is in dB as Web Audio takes it.
+ * passes, in order. `gain` is a factor, a filter's `gain` is in dB as Web Audio takes it. Behind a
+ * Mix (#316) it is two sounds, `b` making up the share `mix` of what goes on through `stages`.
  */
-export interface FxChain {
-  pitch: number;
-  stages: readonly FxStage[];
-}
+export type FxChain =
+  | { pitch: number; stages: readonly FxStage[] }
+  | { mix: number; a: FxChain; b: FxChain; stages: readonly FxStage[] };
 
 export const DRY: FxChain = { pitch: 0, stages: [] };
 
@@ -41,10 +41,9 @@ export type FxStagePlan =
   | { kind: 'filter'; filter: FxFilter; frequency: number; gain: FxAmount; q: number };
 
 /** A chain with its parameters still open: compiled once, filled in for every voice by `fxChainOf`. */
-export interface FxPlan {
-  pitch: readonly FxAmount[];
-  stages: readonly FxStagePlan[];
-}
+export type FxPlan =
+  | { pitch: readonly FxAmount[]; stages: readonly FxStagePlan[] }
+  | { mix: FxAmount; a: FxPlan; b: FxPlan; stages: readonly FxStagePlan[] };
 
 export const EMPTY_PLAN: FxPlan = { pitch: [], stages: [] };
 
@@ -52,12 +51,13 @@ export const EMPTY_PLAN: FxPlan = { pitch: [], stages: [] };
 export type FxParams = Readonly<Record<string, number>>;
 
 /**
- * A node of a sound FX graph (#311). It never runs: `emit` says what it does to the sound arriving
- * at its one audio input, given the numbers at its other inputs. Input and Output have no effect.
+ * A node of a sound FX graph (#311). It never runs: `emit` says what it does to the sounds arriving
+ * at its audio inputs, by input key, given the numbers at its other inputs. Input and Output have no
+ * effect.
  */
 export interface SoundFxNodeDef extends NodeDef {
   category: 'sound';
-  emit(plan: FxPlan, amounts: Readonly<Record<string, FxAmount>>): FxPlan;
+  emit(sounds: Readonly<Record<string, FxPlan>>, amounts: Readonly<Record<string, FxAmount>>): FxPlan;
 }
 
 export function isSoundFxNodeDef(def: NodeDef): def is SoundFxNodeDef {
@@ -92,6 +92,18 @@ function held(amount: FxAmount | undefined, row: FlatDef<'number'>): FxAmount {
 /** Whether an amount does nothing however the sound is played: a typed-in 0. */
 const isNone = (amount: FxAmount) => amount.param === undefined && amount.value === 0;
 
+const arriving = (sounds: Readonly<Record<string, FxPlan>>) => sounds.in ?? EMPTY_PLAN;
+
+function withStages(plan: FxPlan, stages: readonly FxStagePlan[]): FxPlan {
+  return stages.length === 0 ? plan : { ...plan, stages: [...plan.stages, ...stages] };
+}
+
+/** A pitch is worked out on the sound as it starts, so behind a Mix it goes to both sounds. */
+function withPitch(plan: FxPlan, semitones: FxAmount): FxPlan {
+  if ('mix' in plan) return { ...plan, a: withPitch(plan.a, semitones), b: withPitch(plan.b, semitones) };
+  return { ...plan, pitch: [...plan.pitch, semitones] };
+}
+
 /** A parameter of the Input as the nodes it feeds see it. */
 export function soundFxParamPort(port: DeclaredPort): PortDef {
   return { key: port.key, label: port.key, type: 'number' };
@@ -109,7 +121,7 @@ export const SoundFxInput: SoundFxNodeDef = {
   declaresPorts: 'outputs',
   portKinds: ['number'],
   declaredPort: (port) => soundFxParamPort(port),
-  emit: (plan) => plan,
+  emit: () => EMPTY_PLAN,
 };
 
 export const SoundFxOutput: SoundFxNodeDef = {
@@ -120,7 +132,7 @@ export const SoundFxOutput: SoundFxNodeDef = {
   inputs: [audioIn],
   outputs: [],
   params: [],
-  emit: (plan) => plan,
+  emit: arriving,
 };
 
 const GAIN = numberRow('gain', 'Gain (dB)', -120, 24, 0.5);
@@ -134,9 +146,9 @@ export const SoundFxGain: SoundFxNodeDef = {
   inputs: [audioIn, numberIn(GAIN)],
   outputs: [audioOut],
   params: [],
-  emit: (plan, amounts) => {
+  emit: (sounds, amounts) => {
     const db = held(amounts.gain, GAIN);
-    return isNone(db) ? plan : { ...plan, stages: [...plan.stages, { kind: 'gain', db }] };
+    return withStages(arriving(sounds), isNone(db) ? [] : [{ kind: 'gain', db }]);
   },
 };
 
@@ -147,13 +159,13 @@ export const SoundFxPitch: SoundFxNodeDef = {
   label: 'Pitch',
   category: 'sound',
   description:
-    'Plays the sound higher or lower, in semitones: 12 is an octave up, 0.5 a quarter tone. The sound keeps its length, and the pitch applies to the whole sound wherever the node sits in the line.',
+    'Plays the sound higher or lower, in semitones: 12 is an octave up, 0.5 a quarter tone. The sound keeps its length. The pitch is worked out on the sound as it starts, so an effect before the Pitch acts as if it came after it.',
   inputs: [audioIn, numberIn(SEMITONES)],
   outputs: [audioOut],
   params: [],
-  emit: (plan, amounts) => {
+  emit: (sounds, amounts) => {
     const semitones = held(amounts.semitones, SEMITONES);
-    return isNone(semitones) ? plan : { ...plan, pitch: [...plan.pitch, semitones] };
+    return isNone(semitones) ? arriving(sounds) : withPitch(arriving(sounds), semitones);
   },
 };
 
@@ -179,14 +191,35 @@ export const SoundFxEq: SoundFxNodeDef = {
   inputs: [audioIn, ...EQ_ROWS.map(numberIn)],
   outputs: [audioOut],
   params: [],
-  emit: (plan, amounts) => {
+  emit: (sounds, amounts) => {
     const stages: FxStagePlan[] = [];
     EQ_BANDS.forEach((band, i) => {
       const gain = held(amounts[band.key], EQ_ROWS[i]!);
       if (!isNone(gain))
         stages.push({ kind: 'filter', filter: band.filter, frequency: band.frequency, gain, q: PEAK_Q });
     });
-    return stages.length === 0 ? plan : { ...plan, stages: [...plan.stages, ...stages] };
+    return withStages(arriving(sounds), stages);
+  },
+};
+
+const MIX = { ...numberRow('mix', 'Mix', 0, 1, 0.01), default: 0.5 };
+
+export const SoundFxMix: SoundFxNodeDef = {
+  type: 'SoundFxMix',
+  label: 'Mix',
+  category: 'sound',
+  description:
+    'Blends two sounds: at 0 only a is heard, at 1 only b, at 0.5 half of each. Both ways can start at the same Input, each with its own effects, its own Pitch included.',
+  inputs: [{ key: 'a', label: 'a', type: 'audio' }, { key: 'b', label: 'b', type: 'audio' }, numberIn(MIX)],
+  outputs: [audioOut],
+  params: [],
+  emit: (sounds, amounts) => {
+    const mix = held(amounts.mix, MIX);
+    const a = sounds.a ?? EMPTY_PLAN;
+    const b = sounds.b ?? EMPTY_PLAN;
+    if (mix.param === undefined && mix.value <= 0) return a;
+    if (mix.param === undefined && mix.value >= 1) return b;
+    return { mix, a, b, stages: [] };
   },
 };
 
@@ -196,6 +229,7 @@ export const SOUND_FX_NODES: readonly SoundFxNodeDef[] = [
   SoundFxGain,
   SoundFxPitch,
   SoundFxEq,
+  SoundFxMix,
 ];
 
 /** An amount with the value handed in for its parameter, held to its row; a 0 there does nothing. */
@@ -205,8 +239,7 @@ function amountOf(amount: FxAmount, params: FxParams): number {
   return Math.max(amount.min ?? -Infinity, Math.min(amount.max ?? Infinity, given));
 }
 
-/** The chain one voice is sent along: the plan with every parameter filled in. */
-export function fxChainOf(plan: FxPlan, params: FxParams = {}): FxChain {
+function stagesOf(plan: FxPlan, params: FxParams): FxStage[] {
   const stages: FxStage[] = [];
   for (const stage of plan.stages) {
     if (stage.kind === 'gain') {
@@ -217,6 +250,23 @@ export function fxChainOf(plan: FxPlan, params: FxParams = {}): FxChain {
     const gain = amountOf(stage.gain, params);
     if (gain !== 0) stages.push({ ...stage, gain });
   }
-  const pitch = plan.pitch.reduce((sum, amount) => sum + amountOf(amount, params), 0);
-  return { pitch, stages };
+  return stages;
+}
+
+/**
+ * The chain one voice is sent along: the plan with every parameter filled in. A Mix that ends at 0
+ * or 1 is left out with the sound it silences, so that one is not even started.
+ */
+export function fxChainOf(plan: FxPlan, params: FxParams = {}): FxChain {
+  const stages = stagesOf(plan, params);
+  if (!('mix' in plan)) {
+    const pitch = plan.pitch.reduce((sum, amount) => sum + amountOf(amount, params), 0);
+    return { pitch, stages };
+  }
+  const mix = amountOf(plan.mix, params);
+  if (mix <= 0 || mix >= 1) {
+    const heard = fxChainOf(mix <= 0 ? plan.a : plan.b, params);
+    return { ...heard, stages: [...heard.stages, ...stages] };
+  }
+  return { mix, a: fxChainOf(plan.a, params), b: fxChainOf(plan.b, params), stages };
 }

src/viewport/SoundLibrary.ts

@@ -6,15 +6,29 @@ const DEFAULT_BASE = '/game-maker/data/sounds';
 /** How many pitched copies are kept at once. */
 const SHIFTED_KEPT = 32;
 
-/** One sound in flight: its source, its own gain, the file it came from and who is making it. */
+/**
+ * One sound in flight: its sources, its own gain, the file it came from and who is making it. A
+ * voice has a source per pitch its sound FX plays the file at (#316), started together and as long
+ * as each other.
+ */
 interface Voice {
   file: string;
   /** The behavior this voice belongs to; `''` for one nothing owns (see `AudioSink.play`). */
   owner: string;
-  source: AudioBufferSourceNode;
+  sources: AudioBufferSourceNode[];
   gain: GainNode;
 }
 
+/** What one voice is wired from: the file, and its sources by pitch so far. */
+interface Wiring {
+  context: AudioContext;
+  file: string;
+  buffer: AudioBuffer;
+  loop: boolean;
+  sources: Map<number, AudioBufferSourceNode>;
+  stages: number;
+}
+
 /**
  * Plays uploaded sounds, keyed by file name – the texture library's twin on the audio side. Files
  * are content addressed, so a decoded buffer can never go stale and nothing is evicted.
@@ -157,7 +171,7 @@ export class SoundLibrary implements AudioSink {
 
   /** Ends these voices now: `onended` only fires next frame and the counts have to be right here. */
   private silence(going: Voice[]): void {
-    for (const voice of going) voice.source.stop();
+    for (const voice of going) for (const source of voice.sources) source.stop();
     this.voices = this.voices.filter((voice) => !going.includes(voice));
     this.stopped += going.length;
   }
@@ -174,30 +188,58 @@ export class SoundLibrary implements AudioSink {
     // A context starts suspended where the page has not been interacted with; without this the
     // voice would run silently and end as if it had played.
     if (context.state === 'suspended') void context.resume();
-    const source = context.createBufferSource();
-    const played = fx.pitch === 0 ? buffer : this.pitched(context, file, buffer, fx.pitch, loop);
-    source.buffer = played;
-    source.loop = loop;
     const gain = context.createGain();
     gain.gain.value = Math.max(0, Math.min(1, volume));
+    const wiring: Wiring = { context, file, buffer, loop, sources: new Map(), stages: 0 };
+    this.wire(wiring, fx).connect(gain).connect(context.destination);
+    const sources = [...wiring.sources.values()];
     this.last = {
       file,
       volume: gain.gain.value,
       loop,
-      pitch: fx.pitch,
-      stages: fx.stages.length,
-      duration: played.duration,
-    };
-    let tail: AudioNode = source;
-    for (const stage of fx.stages) tail = tail.connect(fxNode(context, stage));
-    tail.connect(gain).connect(context.destination);
-    const voice: Voice = { file, owner, source, gain };
-    source.onended = () => {
-      this.voices = this.voices.filter((other) => other !== voice);
+      pitches: [...wiring.sources.keys()],
+      stages: wiring.stages,
+      duration: sources[0]!.buffer!.duration,
     };
+    const voice: Voice = { file, owner, sources, gain };
+    let playing = sources.length;
+    for (const source of sources)
+      source.onended = () => {
+        if (--playing === 0) this.voices = this.voices.filter((other) => other !== voice);
+      };
     this.voices.push(voice);
     this.started += 1;
-    source.start();
+    for (const source of sources) source.start();
+  }
+
+  /**
+   * The nodes a chain becomes, up to the one that carries its sound on. Two ways at the same pitch
+   * share a source; a Mix weighs each of its sounds by its share before they are summed.
+   */
+  private wire(wiring: Wiring, chain: FxChain): AudioNode {
+    const { context } = wiring;
+    let tail: AudioNode;
+    if ('mix' in chain) {
+      tail = context.createGain();
+      this.wire(wiring, chain.a)
+        .connect(gainNode(context, 1 - chain.mix))
+        .connect(tail);
+      this.wire(wiring, chain.b).connect(gainNode(context, chain.mix)).connect(tail);
+    } else tail = this.source(wiring, chain.pitch);
+    for (const stage of chain.stages) tail = tail.connect(fxNode(context, stage));
+    wiring.stages += chain.stages.length;
+    return tail;
+  }
+
+  private source(wiring: Wiring, pitch: number): AudioBufferSourceNode {
+    const held = wiring.sources.get(pitch);
+    if (held) return held;
+    const { context, file, buffer, loop } = wiring;
+    const source = context.createBufferSource();
+    source.buffer = pitch === 0 ? buffer : this.pitched(context, file, buffer, pitch, loop);
+    source.loop = loop;
+    wiring.sources.set(pitch, source);
+    return source;
   }
 
   /**
@@ -244,19 +286,21 @@ interface LastVoice {
   file: string;
   volume: number;
   loop: boolean;
-  /** Semitones, and how many gain and filter nodes it passed. */
-  pitch: number;
+  /** Semitones of each source, one per pitch (#316), and how many gain and filter nodes it passed. */
+  pitches: number[];
   stages: number;
   /** Seconds of what was played: a pitched voice is as long as the sound itself (#312). */
   duration: number;
 }
 
+function gainNode(context: AudioContext, level: number): GainNode {
+  const gain = context.createGain();
+  gain.gain.value = level;
+  return gain;
+}
+
 function fxNode(context: AudioContext, stage: FxStage): AudioNode {
-  if (stage.kind === 'gain') {
-    const gain = context.createGain();
-    gain.gain.value = stage.gain;
-    return gain;
-  }
+  if (stage.kind === 'gain') return gainNode(context, stage.gain);
   const filter = context.createBiquadFilter();
   filter.type = stage.filter;
   filter.frequency.value = stage.frequency;