adminandClaude Opus 5.5
ed2b58b3c0
A sound was fetched and decoded on its first Play sound. That play started late, its beginning was not heard, and a graph that tuned the voice in the meantime had moved on by the time it started. Images and fonts were already loaded when a project opens. Sounds now are too. - projectFiles lists the sounds of a project, and preloadProjectFiles hands them to the new soundLibrary.preload. That covers the editor and the published game. - Preloading opens no audio device. It decodes with an OfflineAudioContext at 48 kHz. The real AudioContext is still made on the first play, and a source resamples where the device runs at another rate. - A run started right after opening a project can still find a sound loading. It then starts late as before. Tests: - sound-library: a preload makes only the offline context, and a play after it starts at once, opening the device only then. - sound-library + project-files: 18 tests, green. - e2e 34 on melektaus-atx: ALL PASSED, 30 checks. - Counter-check: decoding with the real AudioContext again fails the new test. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
457 lines
16 KiB
TypeScript
457 lines
16 KiB
TypeScript
/**
|
||
* The library between `Play sound` and the audio device. Everything here is about the gap between
|
||
* asking for a sound and being able to play it: the bytes are fetched and decoded on first use, so
|
||
* a stop can land while a play is still in flight.
|
||
*
|
||
* The device is stubbed – nothing makes a noise in a unit test, and Web Audio does not exist here.
|
||
*/
|
||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||
import type { FxChain, FxPlan, FxVoice } from '../../src/core/soundfx/nodes';
|
||
import { SoundLibrary } from '../../src/viewport/SoundLibrary';
|
||
|
||
const FILE = 'a.wav';
|
||
const OTHER = 'b.wav';
|
||
|
||
/** An AudioParam that remembers where it was sent to glide, and how often. */
|
||
interface FakeParam {
|
||
value: number;
|
||
glides: number[];
|
||
setTargetAtTime: (value: number) => void;
|
||
}
|
||
|
||
function param(value: number): FakeParam {
|
||
const made: FakeParam = { value, glides: [], setTargetAtTime: (to) => made.glides.push(to) };
|
||
return made;
|
||
}
|
||
|
||
interface FakeSource {
|
||
buffer: unknown;
|
||
loop: boolean;
|
||
playbackRate: FakeParam;
|
||
onended: (() => void) | null;
|
||
started: boolean;
|
||
stopped: boolean;
|
||
connect: (target: unknown) => unknown;
|
||
start: () => void;
|
||
stop: () => void;
|
||
}
|
||
|
||
const sources: FakeSource[] = [];
|
||
/** Every `connect` made, as "from > to" by the kind of node – the order a voice is wired in. */
|
||
const wires: string[] = [];
|
||
/** The level of every gain node made, in the order they were made. */
|
||
const gains: FakeParam[] = [];
|
||
const filters: {
|
||
type: string;
|
||
frequency: { value: number };
|
||
gain: FakeParam;
|
||
Q: { value: number };
|
||
}[] = [];
|
||
|
||
/** A node that logs where it is connected to and hands the target back, as Web Audio's does. */
|
||
function wired<T extends object>(
|
||
kind: string,
|
||
node: T,
|
||
): T & { kind: string; connect: (to: unknown) => unknown } {
|
||
return {
|
||
...node,
|
||
kind,
|
||
connect: (to: unknown) => {
|
||
wires.push(`${kind} > ${(to as { kind?: string }).kind ?? '?'}`);
|
||
return to;
|
||
},
|
||
};
|
||
}
|
||
|
||
/** Resolves the pending decodes by hand, so a test decides when the bytes arrive. */
|
||
let decodes: (() => void)[] = [];
|
||
|
||
/** A mono buffer as Web Audio hands it out: a length, a rate and the samples. */
|
||
function fakeBuffer(_channels: number, length: number, sampleRate: number) {
|
||
const data = new Float32Array(length);
|
||
return {
|
||
numberOfChannels: 1,
|
||
length,
|
||
sampleRate,
|
||
duration: length / sampleRate,
|
||
getChannelData: () => data,
|
||
};
|
||
}
|
||
|
||
function fakeContext() {
|
||
return {
|
||
state: 'running',
|
||
currentTime: 0,
|
||
destination: { kind: 'destination' },
|
||
resume: () => Promise.resolve(),
|
||
suspend: () => Promise.resolve(),
|
||
decodeAudioData: () => new Promise((resolve) => decodes.push(() => resolve(fakeBuffer(1, 1200, 800)))),
|
||
createBuffer: fakeBuffer,
|
||
createBufferSource: () => {
|
||
const source: FakeSource = wired('source', {
|
||
buffer: null,
|
||
loop: false,
|
||
playbackRate: param(1),
|
||
onended: null,
|
||
started: false,
|
||
stopped: false,
|
||
start: () => {
|
||
source.started = true;
|
||
},
|
||
stop: () => {
|
||
source.stopped = true;
|
||
},
|
||
});
|
||
sources.push(source);
|
||
return source;
|
||
},
|
||
createGain: () => {
|
||
const gain = wired('gain', { gain: param(1) });
|
||
gains.push(gain.gain);
|
||
return gain;
|
||
},
|
||
createBiquadFilter: () => {
|
||
const filter = wired('filter', {
|
||
type: '',
|
||
frequency: { value: 0 },
|
||
gain: param(0),
|
||
Q: { value: 0 },
|
||
});
|
||
filters.push(filter);
|
||
return filter;
|
||
},
|
||
};
|
||
}
|
||
|
||
/** Lets the promise chain inside `play` run to the point where it would start a voice. */
|
||
const settle = () => new Promise((resolve) => setTimeout(resolve, 0));
|
||
|
||
beforeEach(() => {
|
||
sources.length = 0;
|
||
wires.length = 0;
|
||
filters.length = 0;
|
||
gains.length = 0;
|
||
decodes = [];
|
||
vi.stubGlobal(
|
||
'AudioContext',
|
||
class {
|
||
constructor() {
|
||
return fakeContext();
|
||
}
|
||
},
|
||
);
|
||
vi.stubGlobal('fetch', () =>
|
||
Promise.resolve({ ok: true, arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)) }),
|
||
);
|
||
});
|
||
|
||
afterEach(() => {
|
||
vi.unstubAllGlobals();
|
||
});
|
||
|
||
/** A chain as the plan of an FX with nothing left open, so a test can say what it wants to hear. */
|
||
function planOf(chain: FxChain): FxPlan {
|
||
const stages = chain.stages.map((stage) =>
|
||
stage.kind === 'gain'
|
||
? { kind: 'gain' as const, db: { value: 20 * Math.log10(stage.gain) } }
|
||
: { ...stage, gain: { value: stage.gain } },
|
||
);
|
||
if ('mix' in chain) return { mix: { value: chain.mix }, a: planOf(chain.a), b: planOf(chain.b), stages };
|
||
return { pitch: chain.pitch === 0 ? [] : [{ value: chain.pitch }], stages };
|
||
}
|
||
|
||
/** Plays `file` and lets its bytes arrive, unless `hold` keeps the decode pending; gives the voice. */
|
||
async function play(
|
||
library: SoundLibrary,
|
||
file = FILE,
|
||
{ hold = false, owner = '' } = {},
|
||
fx?: FxChain | FxVoice,
|
||
): Promise<number> {
|
||
const voice = library.play(
|
||
file,
|
||
1,
|
||
false,
|
||
owner,
|
||
fx && ('plan' in fx ? fx : { plan: planOf(fx), params: {} }),
|
||
);
|
||
await settle();
|
||
if (!hold) await arrive();
|
||
return voice;
|
||
}
|
||
|
||
async function arrive(): Promise<void> {
|
||
for (const resolve of decodes.splice(0)) resolve();
|
||
await settle();
|
||
}
|
||
|
||
describe('the sound library', () => {
|
||
it('fetches and decodes a file once, however often it is played', async () => {
|
||
const library = new SoundLibrary();
|
||
await play(library);
|
||
await play(library);
|
||
expect(library.stats()).toMatchObject({ started: 2, decoded: 1, playing: 2 });
|
||
// Two voices of one file, both running: plays overlap rather than replacing each other.
|
||
expect(sources.filter((s) => s.started)).toHaveLength(2);
|
||
});
|
||
|
||
it('loads ahead of the first play without opening the device, so that play starts at once', async () => {
|
||
const opened: string[] = [];
|
||
vi.stubGlobal(
|
||
'AudioContext',
|
||
class {
|
||
constructor() {
|
||
opened.push('device');
|
||
return fakeContext();
|
||
}
|
||
},
|
||
);
|
||
vi.stubGlobal(
|
||
'OfflineAudioContext',
|
||
class {
|
||
constructor() {
|
||
opened.push('offline');
|
||
return fakeContext();
|
||
}
|
||
},
|
||
);
|
||
const library = new SoundLibrary();
|
||
library.preload([FILE, OTHER, '']);
|
||
await settle();
|
||
await arrive();
|
||
expect(opened).toEqual(['offline']);
|
||
expect(library.stats()).toMatchObject({ decoded: 2, started: 0 });
|
||
// No await: the voice runs in the frame that asked for it, not once the bytes are in.
|
||
const voice = library.play(FILE, 1, false);
|
||
expect(library.stats()).toMatchObject({ started: 1, playing: 1 });
|
||
expect(library.tune(voice, {})).toBe(true);
|
||
expect(opened).toEqual(['offline', 'device']);
|
||
});
|
||
|
||
it('remembers what it last started, so a muted check can see it at all', async () => {
|
||
const library = new SoundLibrary();
|
||
library.play(FILE, 0.25, true);
|
||
await settle();
|
||
for (const resolve of decodes.splice(0)) resolve();
|
||
await settle();
|
||
expect(library.stats().last).toEqual({
|
||
file: FILE,
|
||
volume: 0.25,
|
||
loop: true,
|
||
pitches: [0],
|
||
stages: 0,
|
||
duration: 1.5,
|
||
});
|
||
});
|
||
|
||
it('plays a dry voice straight into its volume', async () => {
|
||
await play(new SoundLibrary());
|
||
expect(wires).toEqual(['source > gain', 'gain > destination']);
|
||
expect(sources[0]?.playbackRate.value).toBe(1);
|
||
});
|
||
|
||
it('sends a voice through its sound FX in order (#311)', async () => {
|
||
const library = new SoundLibrary();
|
||
const fx: FxChain = {
|
||
pitch: 12,
|
||
stages: [
|
||
{ kind: 'gain', gain: 0.5 },
|
||
{ kind: 'filter', filter: 'lowshelf', frequency: 80, gain: 6, q: 1 },
|
||
],
|
||
};
|
||
await play(library, FILE, {}, fx);
|
||
expect(wires).toEqual(['source > gain', 'gain > filter', 'filter > gain', 'gain > destination']);
|
||
expect(filters).toMatchObject([
|
||
{ type: 'lowshelf', frequency: { value: 80 }, gain: { value: 6 }, Q: { value: 1 } },
|
||
]);
|
||
expect(library.stats().last).toMatchObject({ pitches: [12], stages: 2 });
|
||
});
|
||
|
||
it('pitches a voice on a copy of its length rather than by speed, and keeps the copy (#312)', async () => {
|
||
const library = new SoundLibrary();
|
||
await play(library);
|
||
await play(library, FILE, {}, { pitch: 7, stages: [] });
|
||
await play(library, FILE, {}, { pitch: 7, stages: [] });
|
||
await play(library, FILE, {}, { pitch: -3.5, stages: [] });
|
||
const [dry, up, again, down] = sources.map((s) => s.buffer as { length: number; duration: number });
|
||
expect(sources.map((s) => s.playbackRate.value)).toEqual([1, 1, 1, 1]);
|
||
expect(up).not.toBe(dry);
|
||
expect(again).toBe(up);
|
||
expect(down).not.toBe(up);
|
||
expect([up!.duration, down!.duration]).toEqual([dry!.duration, dry!.duration]);
|
||
});
|
||
|
||
it('mixes two sounds by their shares, a source per way, started and stopped together (#316)', async () => {
|
||
const library = new SoundLibrary();
|
||
await play(
|
||
library,
|
||
FILE,
|
||
{},
|
||
{
|
||
mix: 0.25,
|
||
a: { pitch: 12, stages: [] },
|
||
b: { pitch: 0, stages: [{ kind: 'gain', gain: 0.5 }] },
|
||
stages: [{ kind: 'filter', filter: 'lowshelf', frequency: 80, gain: 6, q: 1 }],
|
||
},
|
||
);
|
||
expect(wires).toEqual([
|
||
'source > gain',
|
||
'gain > gain',
|
||
'source > gain',
|
||
'gain > gain',
|
||
'gain > gain',
|
||
'gain > filter',
|
||
'filter > gain',
|
||
'gain > destination',
|
||
]);
|
||
expect(gains.map((g) => g.value)).toEqual([1, 1, 0.75, expect.closeTo(0.5), 0.25]);
|
||
expect(sources.map((s) => s.started)).toEqual([true, true]);
|
||
expect(library.stats()).toMatchObject({ playing: 1, started: 1, last: { pitches: [12, 0], stages: 2 } });
|
||
|
||
sources[0]!.onended!();
|
||
expect(library.stats().playing).toBe(1);
|
||
library.stop(FILE);
|
||
expect(sources.map((s) => s.stopped)).toEqual([true, true]);
|
||
expect(library.stats()).toMatchObject({ playing: 0, stopped: 1 });
|
||
});
|
||
|
||
it('gives two ways at the same pitch a source each, so each can glide on its own (#315)', async () => {
|
||
const library = new SoundLibrary();
|
||
await play(
|
||
library,
|
||
FILE,
|
||
{},
|
||
{
|
||
mix: 0.5,
|
||
a: { pitch: 0, stages: [] },
|
||
b: { pitch: 0, stages: [{ kind: 'gain', gain: 0.5 }] },
|
||
stages: [],
|
||
},
|
||
);
|
||
expect(sources).toHaveLength(2);
|
||
expect(sources[0]!.buffer).toBe(sources[1]!.buffer);
|
||
expect(wires.filter((w) => w.startsWith('source'))).toEqual(['source > gain', 'source > gain']);
|
||
});
|
||
|
||
describe('a parameter changed while the voice plays (#315)', () => {
|
||
/** Pitch on a, a gain on b, the share between them – all three open – and a band after the Mix. */
|
||
const open: FxPlan = {
|
||
mix: { value: 0.25, param: 'blend', min: 0, max: 1 },
|
||
a: { pitch: [{ value: 0, param: 'depth', min: -64, max: 64 }], stages: [] },
|
||
b: { pitch: [], stages: [{ kind: 'gain', db: { value: 0, param: 'loud', min: -120, max: 24 } }] },
|
||
stages: [{ kind: 'filter', filter: 'lowshelf', frequency: 80, gain: { value: 6 }, q: 1 }],
|
||
};
|
||
|
||
it('glides every knob it moves, a pitch by speed, and leaves the rest alone', async () => {
|
||
const library = new SoundLibrary();
|
||
const voice = await play(library, FILE, { owner: 'me' }, { plan: open, params: {} });
|
||
// Made in the order: the voice, the Mix, share of a, the gain on b, share of b.
|
||
const [, , shareA, loud, shareB] = gains;
|
||
expect([shareA!.value, loud!.value, shareB!.value]).toEqual([0.75, 1, 0.25]);
|
||
|
||
expect(library.tune(voice, { depth: 12, loud: -6, blend: 0.5, nothing: 3 })).toBe(true);
|
||
expect(sources[0]!.playbackRate.glides).toEqual([2]);
|
||
expect(sources[1]!.playbackRate.glides).toEqual([]);
|
||
expect(loud!.glides).toEqual([expect.closeTo(10 ** (-6 / 20))]);
|
||
expect([shareA!.glides, shareB!.glides]).toEqual([[0.5], [0.5]]);
|
||
expect(filters[0]!.gain.glides).toEqual([]);
|
||
expect(library.stats().tuned).toBe(1);
|
||
|
||
// Only what changed is sent again; one not given keeps its value rather than its default.
|
||
library.tune(voice, { depth: 12, blend: 1 });
|
||
expect(sources[0]!.playbackRate.glides).toEqual([2]);
|
||
expect(loud!.glides).toHaveLength(1);
|
||
expect([shareA!.glides, shareB!.glides]).toEqual([
|
||
[0.5, 0],
|
||
[0.5, 1],
|
||
]);
|
||
});
|
||
|
||
it('reaches the one voice it is given – a detached one too – and none that has ended', async () => {
|
||
const library = new SoundLibrary();
|
||
const voices = [
|
||
await play(library, FILE, { owner: 'me' }, { plan: open, params: {} }),
|
||
await play(library, FILE, { owner: 'me' }, { plan: open, params: {} }),
|
||
await play(library, OTHER, { owner: 'me' }, { plan: open, params: {} }),
|
||
await play(library, FILE, {}, { plan: open, params: {} }),
|
||
];
|
||
expect(new Set(voices).size).toBe(4);
|
||
expect(voices).not.toContain(0);
|
||
library.tune(voices[1]!, { depth: 12 });
|
||
library.tune(voices[3]!, { depth: 12 });
|
||
// Two sources a voice, the first of each the pitched way.
|
||
expect(sources.map((s) => s.playbackRate.glides.length)).toEqual([0, 0, 1, 0, 0, 0, 1, 0]);
|
||
|
||
library.stop(FILE);
|
||
expect(library.tune(voices[1]!, { depth: 3 })).toBe(false);
|
||
expect(library.tune(0, { depth: 3 })).toBe(false);
|
||
expect(library.stats().tuned).toBe(2);
|
||
});
|
||
|
||
it('starts a voice still waiting on its bytes with the values handed in meanwhile', async () => {
|
||
const library = new SoundLibrary();
|
||
const voice = await play(
|
||
library,
|
||
FILE,
|
||
{ hold: true, owner: 'me' },
|
||
{ plan: open, params: { blend: 0.5 } },
|
||
);
|
||
expect(library.tune(voice, { depth: 7 })).toBe(true);
|
||
await arrive();
|
||
expect(library.stats().last?.pitches).toEqual([7, 0]);
|
||
expect(gains.map((g) => g.value)).toContain(0.5);
|
||
expect(sources[0]!.playbackRate.glides).toEqual([]);
|
||
});
|
||
});
|
||
|
||
it('a stop while the bytes are in flight keeps the sound from starting afterwards', async () => {
|
||
const library = new SoundLibrary();
|
||
await play(library, FILE, { hold: true });
|
||
library.stop(FILE);
|
||
for (const resolve of decodes.splice(0)) resolve();
|
||
await settle();
|
||
expect(library.stats()).toMatchObject({ started: 0, playing: 0 });
|
||
});
|
||
|
||
it('but a stop before a play does not: an earlier silence is not a standing veto', async () => {
|
||
const library = new SoundLibrary();
|
||
// Exactly what ending a run does – and the next run has to be able to play that sound again,
|
||
// even though its bytes are still unread at that point.
|
||
library.stop('');
|
||
library.stop(FILE);
|
||
await play(library);
|
||
expect(library.stats()).toMatchObject({ started: 1, playing: 1 });
|
||
});
|
||
|
||
it('stopping everything silences every voice, and only the named file otherwise', async () => {
|
||
const library = new SoundLibrary();
|
||
await play(library, FILE);
|
||
await play(library, OTHER);
|
||
library.stop(FILE);
|
||
expect(library.stats()).toMatchObject({ playing: 1, stopped: 1 });
|
||
|
||
library.stopAll();
|
||
expect(library.stats()).toMatchObject({ playing: 0, stopped: 2 });
|
||
expect(sources.every((s) => s.stopped)).toBe(true);
|
||
});
|
||
|
||
it('a voice that ends by itself leaves the count, without being stopped', async () => {
|
||
const library = new SoundLibrary();
|
||
await play(library);
|
||
sources[0]?.onended?.();
|
||
expect(library.stats()).toMatchObject({ playing: 0, started: 1, stopped: 0 });
|
||
});
|
||
|
||
it('a file that cannot be loaded fails once and is never asked for again', async () => {
|
||
const library = new SoundLibrary();
|
||
vi.stubGlobal('fetch', () => Promise.resolve({ ok: false, status: 404 }));
|
||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||
expect(await library.load(FILE)).toBeNull();
|
||
expect(await library.load(FILE)).toBeNull();
|
||
expect(library.stats()).toMatchObject({ failed: 1, started: 0 });
|
||
|
||
library.play(FILE, 1, false);
|
||
await settle();
|
||
expect(library.stats().started).toBe(0);
|
||
});
|
||
});
|