Stream long sounds from an Opus copy instead of decoding them (#349)
A sound was decoded whole as soon as the project opened, in the editor and in a released game: five minutes of stereo held some 115 MB for as long as the page was open. A sound with Stream on now plays from a copy as Opus in Ogg, fetched a piece at a time and decoded a few seconds ahead by WebCodecs, so a game holds a few megabytes for it. - The Stream switch is a row of the sound, off by default. The Sound tab suggests it from a minute on and says how far the copy has got. - StreamKeeper makes the copies in the browser, a moment after the last change, one at a time: the sound's own and one per clip of its cut, fades and Reverse baked in. A copy that is no longer wanted is taken off and deleted once the project is saved without it. - streamTable says which played names stream from which copy. A clip without a fitting copy, a Sample and a Reverb impulse stay decoded. A sound with Stream on is never decoded ahead. - The page fetches by Range and takes the whole file from a server that ignores it, as the CI's PHP server does. It keeps the bytes and the first two seconds decoded, the rest goes four seconds ahead. - A jump decodes 32 packets before the place first. Measured in Chrome, what follows is then the same to the bit as decoding from the start. With 4 packets it was 0.06 off. - The stream-player on the FX worklet reads the samples at the rate of the context along a Catmull-Rom curve. What comes late is silent and skipped once it is there, so a clip keeps its beat. - The opening stretch goes with the player as it is made. Sent by message it came a few milliseconds late, and 557 samples of the attack were lost. - Pitch on a streamed sound is a shifter after the player, as on a bus: the length stays and it is heard 42.7 ms late. - Bakes of patterns and songs (#340) always stream now. - The song timeline draws a streamed sound from its copy decoded at 8 kHz, keeping only the peaks. Tests: - Gate: CHECK EXIT 0, 280 files, 4312 tests (31 new: ogg +3, stream-queue 6, streams 11, streaming 6, sound-library +5). - e2e 34: ALL PASSED, 106 checks (7 new). Five minutes of mono, 4.8 MB as WAV, 3.6 MB as its copy: streamed the renderer grows by 5.8 MB, decoded by 69.6 MB. Offline at 48 kHz, start, loop seam and a reversed clip are the copy decoded whole to the sample (0). - Counter-checks red as they should be: no pre-roll, no catch-up of late samples at the end, no opening stretch, a streamed sound decoded ahead. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
adminandClaude Opus 5.5 committed
1 parent
84cd36f902
commit
2764919bb8
30 files changed
+2443
-146
No files matched your search
+55
-1
@@ -3,7 +3,15 @@
|
||||
* page says it has got to, and the pre-skip and length a decoder trims by.
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { oggCrc, oggOpus, opusHead, preSkipOf, readOggOpus } from '../../src/core/music/ogg';
|
||||
import {
|
||||
OggOpusReader,
|
||||
oggCrc,
|
||||
oggOpus,
|
||||
opusHead,
|
||||
opusPacketSamples,
|
||||
preSkipOf,
|
||||
readOggOpus,
|
||||
} from '../../src/core/music/ogg';
|
||||
|
||||
const packet = (size: number, fill: number) => ({ data: new Uint8Array(size).fill(fill), samples: 960 });
|
||||
|
||||
@@ -65,3 +73,49 @@ describe('Opus in Ogg', () => {
|
||||
expect(preSkipOf(new Uint8Array(19))).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Opus in Ogg as it comes in (#349)', () => {
|
||||
/** A CELT packet of 20 ms at full band: config 31, one frame. */
|
||||
const celt = (size: number, fill: number) => {
|
||||
const data = new Uint8Array(size).fill(fill);
|
||||
data[0] = 31 << 3;
|
||||
return { data, samples: 960 };
|
||||
};
|
||||
const file = oggOpus({
|
||||
channels: 2,
|
||||
preSkip: 312,
|
||||
frames: 300 * 960 - 312,
|
||||
packets: Array.from({ length: 300 }, (_, i) => celt(100 + (i % 300), i % 256)),
|
||||
});
|
||||
|
||||
it('reads a file pushed in pieces as it reads it whole, each packet placed where it starts', () => {
|
||||
const reader = new OggOpusReader();
|
||||
for (let at = 0, size = 1; at < file.length; at += size, size = (size * 7 + 3) % 4000) {
|
||||
reader.push(file.subarray(at, at + size));
|
||||
}
|
||||
const whole = readOggOpus(file);
|
||||
expect(reader.packets.map((p) => p.data)).toEqual(whole.packets);
|
||||
expect(reader.packets.slice(0, 3).map((p) => p.at)).toEqual([0, 960, 1920]);
|
||||
expect(reader.packets.at(-1)!.at).toBe(299 * 960);
|
||||
expect([reader.channels, reader.preSkip, reader.ended]).toEqual([2, 312, true]);
|
||||
});
|
||||
|
||||
it('holds a page back until all of it is there', () => {
|
||||
const reader = new OggOpusReader();
|
||||
reader.push(file.subarray(0, 4000));
|
||||
const seen = reader.packets.length;
|
||||
expect(reader.ended).toBe(false);
|
||||
reader.push(file.subarray(4000));
|
||||
expect(reader.packets.length).toBeGreaterThan(seen);
|
||||
expect(reader.packets).toHaveLength(300);
|
||||
});
|
||||
|
||||
it('counts the samples of a packet from its TOC byte', () => {
|
||||
expect(opusPacketSamples(Uint8Array.of(31 << 3))).toBe(960);
|
||||
expect(opusPacketSamples(Uint8Array.of((28 << 3) | 1))).toBe(240);
|
||||
expect(opusPacketSamples(Uint8Array.of((3 << 3) | 2))).toBe(2 * 2880);
|
||||
expect(opusPacketSamples(Uint8Array.of((13 << 3) | 3, 3))).toBe(3 * 960);
|
||||
expect(opusPacketSamples(Uint8Array.of(12 << 3))).toBe(480);
|
||||
expect(opusPacketSamples(new Uint8Array(0))).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -37,6 +37,8 @@ import {
|
||||
loopsOf,
|
||||
REVERSE_KEY,
|
||||
START_KEY,
|
||||
STREAM_KEY,
|
||||
streamsOf,
|
||||
VOLUME_KEY,
|
||||
volumeOf,
|
||||
} from '../../src/core/properties/sounds';
|
||||
@@ -129,8 +131,10 @@ describe('the sound asset', () => {
|
||||
expect(asset).toMatchObject({ type: 'sound', file: '' });
|
||||
expect(volumeOf(asset.props)).toBe(1);
|
||||
expect(loopsOf(asset.props)).toBe(false);
|
||||
// Decoded whole unless it is switched to stream (#349).
|
||||
expect(streamsOf(asset.props)).toBe(false);
|
||||
// The cut belongs to a clip of the sound (#357), the sound is the whole file.
|
||||
expect(schemaFor('sound').map((def) => def.key)).toEqual([VOLUME_KEY, LOOP_KEY, 'fx']);
|
||||
expect(schemaFor('sound').map((def) => def.key)).toEqual([VOLUME_KEY, LOOP_KEY, STREAM_KEY, 'fx']);
|
||||
expect(CLIP_ROWS.map((def) => def.key)).toEqual([
|
||||
START_KEY,
|
||||
END_KEY,
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* What a streamed sound (#349) is played from on the audio thread: the samples in order, read at the
|
||||
* rate of the context, silent where they come late and skipped once they are there.
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { STREAM_RATE, StreamQueue } from '../../src/core/music/streamQueue';
|
||||
|
||||
const ramp = (from: number, count: number) => Float32Array.from({ length: count }, (_, i) => from + i);
|
||||
const block = (size = 128) => [new Float32Array(size), new Float32Array(size)];
|
||||
|
||||
describe('the queue of a streamed sound', () => {
|
||||
it('plays what it is sent sample for sample at the rate of the stream, across its chunks', () => {
|
||||
const queue = new StreamQueue(2, 1);
|
||||
queue.push([ramp(0, 100), ramp(1000, 100)]);
|
||||
queue.push([ramp(100, 100), ramp(1100, 100)]);
|
||||
queue.end();
|
||||
const out = block();
|
||||
expect(queue.render(out)).toBe(true);
|
||||
expect(Array.from(out[0]!.subarray(98, 102))).toEqual([98, 99, 100, 101]);
|
||||
expect(Array.from(out[1]!.subarray(0, 2))).toEqual([1000, 1001]);
|
||||
expect(queue.render(out)).toBe(false);
|
||||
expect(Array.from(out[0]!.subarray(70, 74))).toEqual([198, 199, 0, 0]);
|
||||
expect(queue.used).toBe(200);
|
||||
});
|
||||
|
||||
it('plays a mono stream on both sides', () => {
|
||||
const queue = new StreamQueue(1, 1);
|
||||
queue.push([ramp(5, 300)]);
|
||||
const out = block();
|
||||
queue.render(out);
|
||||
expect(out[1]![3]).toBe(8);
|
||||
});
|
||||
|
||||
it('reads it at the rate of a context running at 44.1 kHz along a smooth curve', () => {
|
||||
const step = STREAM_RATE / 44100;
|
||||
const queue = new StreamQueue(1, step);
|
||||
const sine = (i: number) => Math.sin((2 * Math.PI * 440 * i) / STREAM_RATE);
|
||||
for (let at = 0; at < 2 * STREAM_RATE; at += 4800)
|
||||
queue.push([Float32Array.from({ length: 4800 }, (_, i) => sine(at + i))]);
|
||||
const out = [new Float32Array(44100)];
|
||||
queue.render(out);
|
||||
let worst = 0;
|
||||
for (let i = 4; i < 44000; i++) worst = Math.max(worst, Math.abs(out[0]![i]! - sine(i * step)));
|
||||
expect(worst).toBeLessThan(1e-4);
|
||||
expect(queue.used).toBe(Math.floor(44100 * step));
|
||||
});
|
||||
|
||||
it('is silent while what is due has not come, and skips as much once it has, so it keeps time', () => {
|
||||
const queue = new StreamQueue(1, 1);
|
||||
queue.push([ramp(0, 64)]);
|
||||
const out = [new Float32Array(128)];
|
||||
queue.render(out);
|
||||
// The last two it had are held for the curve and come late with the rest.
|
||||
expect(out[0]![61]).toBe(61);
|
||||
expect(out[0]![100]).toBe(0);
|
||||
queue.push([ramp(64, 1000)]);
|
||||
expect(queue.used).toBe(128);
|
||||
expect(queue.skipped).toBe(66);
|
||||
queue.render(out);
|
||||
expect(out[0]![0]).toBe(128);
|
||||
});
|
||||
|
||||
it('says it has played out once it has ended and all of it has been read', () => {
|
||||
const queue = new StreamQueue(2, 1);
|
||||
queue.push([ramp(0, 300), ramp(0, 300)]);
|
||||
queue.end();
|
||||
expect(queue.render(block())).toBe(true);
|
||||
expect(queue.render(block())).toBe(true);
|
||||
const out = block();
|
||||
expect(queue.render(out)).toBe(false);
|
||||
expect(Array.from(out[0]!.subarray(42, 46))).toEqual([298, 299, 0, 0]);
|
||||
expect(queue.done).toBe(true);
|
||||
});
|
||||
|
||||
it('skips what came too late even when nothing more comes after it', () => {
|
||||
const queue = new StreamQueue(1, 1);
|
||||
queue.push([ramp(0, 10)]);
|
||||
queue.render(block());
|
||||
queue.end();
|
||||
expect(queue.done).toBe(true);
|
||||
expect(queue.render(block())).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* Sounds streamed rather than decoded whole (#349): which names a game streams from which copy, what
|
||||
* copies are missing or left over, how a played sound is walked through its copy, and that a copy is
|
||||
* kept with the project.
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
CreateAssetCommand,
|
||||
CreateSoundCommand,
|
||||
SetAssetPropertyCommand,
|
||||
} from '../../src/core/commands/assetCommands';
|
||||
import { AddClipCommand, SetClipPropsCommand } from '../../src/core/commands/clipCommands';
|
||||
import type { SongAsset, SoundAsset, SoundStream } from '../../src/core/model/Asset';
|
||||
import { projectFiles } from '../../src/core/model/projectFiles';
|
||||
import { cutName } from '../../src/core/model/sound';
|
||||
import { STREAM_RATE } from '../../src/core/music/streamQueue';
|
||||
import {
|
||||
heldStream,
|
||||
isStreamUpload,
|
||||
playedPlan,
|
||||
StreamCursor,
|
||||
streamTable,
|
||||
streamUploadName,
|
||||
streamWork,
|
||||
} from '../../src/core/music/streams';
|
||||
import { cutOf } from '../../src/core/properties/sounds';
|
||||
import { deserializeProject, serializeProject } from '../../src/core/store/serialize';
|
||||
import { makeSpriteScene } from '../helpers';
|
||||
|
||||
const FILE = `${'a1'.repeat(6)}/${'b2'.repeat(32)}.wav`;
|
||||
const COPY = `${'a1'.repeat(6)}/${'c3'.repeat(32)}.ogg`;
|
||||
const CLIP_COPY = `${'a1'.repeat(6)}/${'d4'.repeat(32)}.ogg`;
|
||||
|
||||
function streamed() {
|
||||
const f = makeSpriteScene('Box');
|
||||
const made = new CreateSoundCommand('Music', FILE);
|
||||
f.stack.execute(made);
|
||||
const sound = made.asset;
|
||||
const clip = new AddClipCommand(sound.id, { start: 1, end: 3 });
|
||||
f.stack.execute(clip);
|
||||
const whole = new AddClipCommand(sound.id, { start: 0, end: 0 });
|
||||
f.stack.execute(whole);
|
||||
const turn = (on: boolean) => f.stack.execute(new SetAssetPropertyCommand(sound.id, 'stream', on));
|
||||
const copy = (from: string, file: string): SoundStream => ({ file, from, frames: 96000, bytes: 4000 });
|
||||
const cut = () => cutName(FILE, cutOf(sound.clips![0]!.props));
|
||||
return { ...f, sound, clipId: clip.clipId, wholeId: whole.clipId, turn, copy, cut };
|
||||
}
|
||||
|
||||
describe('which sounds stream', () => {
|
||||
it('streams nothing while the switch is off, whatever copies there are', () => {
|
||||
const f = streamed();
|
||||
f.sound.stream = f.copy(FILE, COPY);
|
||||
expect(streamTable(f.project).size).toBe(0);
|
||||
});
|
||||
|
||||
it('streams a sound from a copy of its file, a clip from a copy of its cut, a whole-file clip as the sound', () => {
|
||||
const f = streamed();
|
||||
f.turn(true);
|
||||
f.sound.stream = f.copy(FILE, COPY);
|
||||
f.sound.clips![0]!.stream = f.copy(f.cut(), CLIP_COPY);
|
||||
expect([...streamTable(f.project)]).toEqual([
|
||||
[FILE, { file: COPY, frames: 96000 }],
|
||||
[f.cut(), { file: CLIP_COPY, frames: 96000 }],
|
||||
]);
|
||||
});
|
||||
|
||||
it('plays a sound or a clip decoded while its copy is of something else', () => {
|
||||
const f = streamed();
|
||||
f.turn(true);
|
||||
f.sound.stream = f.copy(FILE, COPY);
|
||||
f.sound.clips![0]!.stream = f.copy(f.cut(), CLIP_COPY);
|
||||
f.stack.execute(new SetClipPropsCommand(f.sound.id, f.clipId, { end: 2.5 }));
|
||||
expect([...streamTable(f.project).keys()]).toEqual([FILE]);
|
||||
});
|
||||
|
||||
it('asks for a copy of the sound and of each clip with a cut of its own, and lets them go once it is off', () => {
|
||||
const f = streamed();
|
||||
expect(streamWork(f.project)).toEqual({ jobs: [], drops: [] });
|
||||
f.turn(true);
|
||||
expect(streamWork(f.project).jobs).toEqual([
|
||||
{ assetId: f.sound.id, clipId: null, from: FILE },
|
||||
{ assetId: f.sound.id, clipId: f.clipId, from: f.cut() },
|
||||
]);
|
||||
f.sound.stream = f.copy(FILE, COPY);
|
||||
f.sound.clips![0]!.stream = f.copy(f.cut(), CLIP_COPY);
|
||||
expect(streamWork(f.project)).toEqual({ jobs: [], drops: [] });
|
||||
f.stack.execute(new SetClipPropsCommand(f.sound.id, f.clipId, { reverse: true }));
|
||||
expect(streamWork(f.project).jobs).toEqual([{ assetId: f.sound.id, clipId: f.clipId, from: f.cut() }]);
|
||||
f.turn(false);
|
||||
expect(streamWork(f.project)).toEqual({
|
||||
jobs: [],
|
||||
drops: [
|
||||
{ assetId: f.sound.id, clipId: null },
|
||||
{ assetId: f.sound.id, clipId: f.clipId },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('leaves a sound that streams out of what is decoded ahead, its copy made or not', () => {
|
||||
const f = streamed();
|
||||
expect(projectFiles(f.project).sounds).toEqual([FILE]);
|
||||
f.turn(true);
|
||||
expect(projectFiles(f.project).sounds).toEqual([]);
|
||||
});
|
||||
|
||||
it('streams every bake from itself', () => {
|
||||
const f = streamed();
|
||||
const made = new CreateAssetCommand('song', 'Theme');
|
||||
f.stack.execute(made);
|
||||
(made.asset as SongAsset).bake = {
|
||||
file: COPY,
|
||||
print: 'x',
|
||||
peak: 1,
|
||||
length: 100,
|
||||
loopEnd: 80,
|
||||
head: 100,
|
||||
round: 40,
|
||||
bytes: 9,
|
||||
};
|
||||
expect(streamTable(f.project).get(COPY)).toEqual({ file: COPY, frames: 140 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('walking a copy', () => {
|
||||
it('plays a sound from where it is asked to, round its whole length where it loops', () => {
|
||||
expect(playedPlan(1000, 0, false)).toEqual({ from: 0, until: 1000, loop: null });
|
||||
expect(playedPlan(STREAM_RATE * 2, 2.5, true)).toEqual({
|
||||
from: STREAM_RATE / 2,
|
||||
until: STREAM_RATE * 2,
|
||||
loop: { start: 0, end: STREAM_RATE * 2 },
|
||||
});
|
||||
expect(playedPlan(1000, 1, false).from).toBe(1000);
|
||||
});
|
||||
|
||||
it('reads straight on to the end of a run, then round the loop, starting a new stretch there', () => {
|
||||
const cursor = new StreamCursor({ from: 50, until: 120, loop: { start: 100, end: 130 } });
|
||||
const read: number[][] = [];
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const span = cursor.next(40)!;
|
||||
read.push([span.from, span.to]);
|
||||
}
|
||||
expect(read).toEqual([
|
||||
[50, 90],
|
||||
[90, 120],
|
||||
[100, 130],
|
||||
[100, 130],
|
||||
[100, 130],
|
||||
]);
|
||||
expect(cursor.peek()).toBe(100);
|
||||
});
|
||||
|
||||
it('ends a run without a loop', () => {
|
||||
const cursor = new StreamCursor({ from: 0, until: 10, loop: null });
|
||||
expect(cursor.next(100)).toEqual({ from: 0, to: 10 });
|
||||
expect(cursor.peek()).toBeNull();
|
||||
expect(cursor.next(100)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('a copy kept with the project', () => {
|
||||
it('comes back from a save, and one that is not right is dropped', () => {
|
||||
const f = streamed();
|
||||
f.sound.stream = f.copy(FILE, COPY);
|
||||
f.sound.clips![0]!.stream = f.copy(f.cut(), CLIP_COPY);
|
||||
const back = deserializeProject(serializeProject(f.project));
|
||||
const sound = back.assets.find((a) => a.id === f.sound.id) as SoundAsset;
|
||||
expect(sound.stream).toEqual(f.copy(FILE, COPY));
|
||||
expect(sound.clips![0]!.stream).toEqual(f.copy(f.cut(), CLIP_COPY));
|
||||
expect(sound.clips![1]!.stream).toBeUndefined();
|
||||
expect(heldStream({ ...f.copy(FILE, COPY), frames: 1.5 })).toBeNull();
|
||||
expect(heldStream({ ...f.copy('', COPY) })).toBeNull();
|
||||
expect(heldStream({ ...f.copy(FILE, 'notes.txt') })).toBeNull();
|
||||
});
|
||||
|
||||
it('is uploaded under a mark the sound picker leaves out by', () => {
|
||||
expect(streamUploadName('Music / Intro')).toBe('[stream] Music - Intro.ogg');
|
||||
expect(isStreamUpload(streamUploadName('Music'))).toBe(true);
|
||||
expect(isStreamUpload('Music.ogg')).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
/**
|
||||
* What streaming (#349) needs of a browser, stubbed: a copy as Opus in Ogg served with or without
|
||||
* `Range`, and an `AudioDecoder` whose every sample is where it lies in the stream – so a check sees
|
||||
* to the sample whether what is played is what it should be.
|
||||
*/
|
||||
import { vi } from 'vitest';
|
||||
import { oggOpus } from '../../src/core/music/ogg';
|
||||
|
||||
export const PRE_SKIP = 312;
|
||||
/** Bytes of each packet: big enough that a copy of a few seconds comes in several pieces. */
|
||||
const PACKET_BYTES = 4000;
|
||||
|
||||
/** A copy of `seconds` seconds: CELT packets of 20 ms. */
|
||||
export function fakeCopy(seconds: number, channels = 2): { bytes: Uint8Array; frames: number } {
|
||||
const count = Math.ceil((seconds * 48000 + PRE_SKIP) / 960);
|
||||
const packets = Array.from({ length: count }, () => {
|
||||
const data = new Uint8Array(PACKET_BYTES);
|
||||
data[0] = 31 << 3;
|
||||
return { data, samples: 960 };
|
||||
});
|
||||
const frames = Math.round(seconds * 48000);
|
||||
return { bytes: oggOpus({ channels, preSkip: PRE_SKIP, frames, packets }), frames };
|
||||
}
|
||||
|
||||
export interface FakeServer {
|
||||
/** Every request, as the range it asked for ('' for the whole file). */
|
||||
asked: string[];
|
||||
}
|
||||
|
||||
/** Serves these files by URL ending; `ranges` false answers every request with the whole file. */
|
||||
export function serveCopies(files: Record<string, Uint8Array>, ranges = true): FakeServer {
|
||||
const server: FakeServer = { asked: [] };
|
||||
vi.stubGlobal('fetch', (url: string, init?: { headers?: Record<string, string> }) => {
|
||||
const name = Object.keys(files).find((file) => url.endsWith(file));
|
||||
if (!name) return Promise.resolve({ ok: false, status: 404 });
|
||||
const bytes = files[name]!;
|
||||
const range = init?.headers?.Range ?? '';
|
||||
server.asked.push(range);
|
||||
const [from, to] = range ? range.replace('bytes=', '').split('-').map(Number) : [0, bytes.length - 1];
|
||||
if (!ranges || !range) return Promise.resolve(answer(200, bytes, null));
|
||||
if (from! >= bytes.length) return Promise.resolve(answer(416, new Uint8Array(0), null));
|
||||
const part = bytes.slice(from, Math.min(bytes.length, to! + 1));
|
||||
return Promise.resolve(answer(206, part, `bytes ${from}-${from! + part.length - 1}/${bytes.length}`));
|
||||
});
|
||||
return server;
|
||||
}
|
||||
|
||||
function answer(status: number, bytes: Uint8Array, contentRange: string | null) {
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status,
|
||||
headers: { get: (name: string) => (name.toLowerCase() === 'content-range' ? contentRange : null) },
|
||||
arrayBuffer: () => Promise.resolve(bytes.slice().buffer),
|
||||
};
|
||||
}
|
||||
|
||||
/** Every decoder made, with the timestamps of what it was given and whether it was closed. */
|
||||
export const decoders: { fed: number[]; closed: boolean; channels: number }[] = [];
|
||||
|
||||
/** An `AudioDecoder` that gives back, for each packet, 960 samples counting up from where it starts. */
|
||||
export function stubDecoder(): void {
|
||||
decoders.length = 0;
|
||||
vi.stubGlobal(
|
||||
'EncodedAudioChunk',
|
||||
class {
|
||||
readonly timestamp: number;
|
||||
constructor(init: { timestamp: number }) {
|
||||
this.timestamp = init.timestamp;
|
||||
}
|
||||
},
|
||||
);
|
||||
vi.stubGlobal(
|
||||
'AudioDecoder',
|
||||
class {
|
||||
state = 'unconfigured';
|
||||
private readonly made = { fed: [] as number[], closed: false, channels: 0 };
|
||||
constructor(private readonly init: { output: (data: unknown) => void }) {
|
||||
decoders.push(this.made);
|
||||
}
|
||||
configure(config: { numberOfChannels: number }) {
|
||||
this.state = 'configured';
|
||||
this.made.channels = config.numberOfChannels;
|
||||
}
|
||||
decode(chunk: { timestamp: number }) {
|
||||
const at = Math.round((chunk.timestamp * 48000) / 1e6);
|
||||
this.made.fed.push(at);
|
||||
queueMicrotask(() => {
|
||||
if (this.made.closed) return;
|
||||
this.init.output({
|
||||
numberOfFrames: 960,
|
||||
numberOfChannels: this.made.channels,
|
||||
copyTo: (into: Float32Array, { planeIndex }: { planeIndex: number }) => {
|
||||
for (let i = 0; i < into.length; i++) into[i] = at + i + planeIndex * 1e6;
|
||||
},
|
||||
close: () => {},
|
||||
});
|
||||
});
|
||||
}
|
||||
close() {
|
||||
this.state = 'closed';
|
||||
this.made.closed = true;
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -18,10 +18,11 @@ import {
|
||||
import type { MixBus } from '../../src/core/music/mixer';
|
||||
import type { BakePlay, SequencePlay } from '../../src/core/music/playbacks';
|
||||
import type { ScheduledNote } from '../../src/core/music/Scheduler';
|
||||
import type { LfoOptions, PitchOptions } from '../../src/viewport/fxWorklet';
|
||||
import type { LfoOptions, PitchOptions, StreamOptions } from '../../src/viewport/fxWorklet';
|
||||
import { saturationCurve } from '../../src/core/soundfx/saturation';
|
||||
import { stageNodes } from '../../src/viewport/fxNodes';
|
||||
import { SoundLibrary } from '../../src/viewport/SoundLibrary';
|
||||
import { fakeCopy, PRE_SKIP, serveCopies, stubDecoder } from './fakeStream';
|
||||
|
||||
/** What a test wants to hear: a chain whose pitch, where it is not 0, is a Pitch typed in. */
|
||||
type Chain =
|
||||
@@ -368,7 +369,10 @@ beforeEach(() => {
|
||||
constructor(_context: unknown, name: string, options: FakeWorklet['options']) {
|
||||
if (!moduleLoaded) throw new DOMException('no such processor', 'InvalidStateError');
|
||||
const parameters = new Map(
|
||||
Object.entries(options.parameterData).map(([key, value]) => [key, param(value)] as const),
|
||||
// A stream player (#349) is made without any.
|
||||
Object.entries(
|
||||
(options.parameterData as FakeWorklet['options']['parameterData'] | undefined) ?? {},
|
||||
).map(([key, value]) => [key, param(value)] as const),
|
||||
);
|
||||
const kind =
|
||||
name === 'bit-crusher'
|
||||
@@ -379,7 +383,9 @@ beforeEach(() => {
|
||||
? name
|
||||
: name === 'pitch-shifter'
|
||||
? 'shifter'
|
||||
: 'pitch';
|
||||
: name === 'stream-player'
|
||||
? 'stream'
|
||||
: 'pitch';
|
||||
const node: FakeWorklet = wired(kind, {
|
||||
options,
|
||||
pitch: parameters.get('pitch') ?? param(0),
|
||||
@@ -1983,6 +1989,112 @@ describe('a bake (#340)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('a sound streamed (#349)', () => {
|
||||
const COPY = 'copy.ogg';
|
||||
const BAKED = 'baked.ogg';
|
||||
/** Long enough that the waits below see the feed settle. */
|
||||
const drain = async () => {
|
||||
for (let i = 0; i < 200; i++) await settle();
|
||||
};
|
||||
const streaming = async () => {
|
||||
stubDecoder();
|
||||
const copy = fakeCopy(10);
|
||||
serveCopies({ [COPY]: copy.bytes, [BAKED]: copy.bytes });
|
||||
const library = new SoundLibrary(false);
|
||||
library.setStreams(
|
||||
new Map([
|
||||
[FILE, { file: COPY, frames: copy.frames }],
|
||||
[BAKED, { file: BAKED, frames: copy.frames }],
|
||||
]),
|
||||
);
|
||||
await drain();
|
||||
return library;
|
||||
};
|
||||
const player = () => worklets.find((w) => w.kind === 'stream')!;
|
||||
/** What a player was handed as it was made and sent after, channel 0. */
|
||||
const heard = (node: FakeWorklet) =>
|
||||
[(node.options.processorOptions as unknown as StreamOptions).first, ...node.posted]
|
||||
.filter((chunk): chunk is Float32Array[] => Array.isArray(chunk) && chunk.length > 0)
|
||||
.flatMap((chunk) => Array.from(chunk[0]!));
|
||||
|
||||
it('plays from its copy on a stream player as long as the copy, none of the file decoded', async () => {
|
||||
const library = await streaming();
|
||||
expect(library.duration(FILE)).toBe(10);
|
||||
const voice = library.play(FILE, 1, true);
|
||||
await drain();
|
||||
expect(player().options).toMatchObject({
|
||||
outputChannelCount: [2],
|
||||
processorOptions: { channels: 2, startAt: 0 },
|
||||
});
|
||||
expect(heard(player())[0]).toBe(PRE_SKIP);
|
||||
expect((player().options.processorOptions as unknown as StreamOptions).first[0]).toHaveLength(96000);
|
||||
expect(wires).toEqual(['stream > gain', ...INTO_MASTER]);
|
||||
expect(decodes).toHaveLength(0);
|
||||
expect(library.stats()).toMatchObject({ decoded: 0, playing: 1, streaming: { playing: 1, heads: 2 } });
|
||||
library.stopVoice(voice);
|
||||
clockTo();
|
||||
expect(player().posted.at(-1)).toBe('stop');
|
||||
expect(library.stats()).toMatchObject({ playing: 0, streaming: { playing: 0 } });
|
||||
});
|
||||
|
||||
it('moves its pitch with a shifter of what it plays, as a bus does', async () => {
|
||||
const library = await streaming();
|
||||
await play(library, FILE, {}, { pitch: 7, stages: [] });
|
||||
await drain();
|
||||
expect(wires).toEqual(['stream > shifter', 'shifter > gain', ...INTO_MASTER]);
|
||||
expect(worklets.find((w) => w.kind === 'shifter')!.pitch.value).toBe(7);
|
||||
expect(library.stats()).toMatchObject({ vocoders: 1, last: { pitches: [7] } });
|
||||
});
|
||||
|
||||
it('starts a clip of a song as far into its copy as its offset is', async () => {
|
||||
const library = await streaming();
|
||||
library.playClip(FILE, { at: 0, offset: 12.5, duration: 4, volume: 1, bus: '' });
|
||||
await drain();
|
||||
expect(heard(player())[0]).toBe(PRE_SKIP + 2.5 * 48000);
|
||||
});
|
||||
|
||||
it('plays a sound whose copy cannot be had decoded, as without one', async () => {
|
||||
stubDecoder();
|
||||
vi.stubGlobal('fetch', (url: string) =>
|
||||
Promise.resolve(
|
||||
url.endsWith(COPY)
|
||||
? { ok: false, status: 404 }
|
||||
: { ok: true, arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)) },
|
||||
),
|
||||
);
|
||||
const library = new SoundLibrary(false);
|
||||
library.setStreams(new Map([[FILE, { file: COPY, frames: 480000 }]]));
|
||||
await drain();
|
||||
await play(library);
|
||||
expect(worklets.filter((w) => w.kind === 'stream')).toHaveLength(0);
|
||||
expect(sources.filter((s) => s.started)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('prepares a bake by the head of its file and streams it round its round, to the sample', async () => {
|
||||
const library = await streaming();
|
||||
library.playSequence(
|
||||
'pattern:p',
|
||||
{
|
||||
sequence: { notes: [{ tick: 0, length: 480, velocity: 1, instrument: 'i', note: 60 }] },
|
||||
tempo: 120,
|
||||
length: 1920,
|
||||
loop: { start: 0, end: 1920 },
|
||||
needs: { layers: [], files: [] },
|
||||
strike: () => 0,
|
||||
bake: { file: BAKED, gain: 0.5, playback: { end: null, until: 2, round: { start: 3, end: 5 } } },
|
||||
},
|
||||
'',
|
||||
);
|
||||
await drain();
|
||||
expect(decodes).toHaveLength(0);
|
||||
expect(sources).toHaveLength(0);
|
||||
const sent = heard(player());
|
||||
expect(sent[2 * 48000 - 1]).toBe(PRE_SKIP + 2 * 48000 - 1);
|
||||
expect(sent[2 * 48000]).toBe(PRE_SKIP + 3 * 48000);
|
||||
expect(library.playbacks()[0]?.baked).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('a sound FX on a bus (#346)', () => {
|
||||
/** One frame of the shifter at the fake context's 48 kHz. */
|
||||
const LATE = 2048 / 48000;
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
/**
|
||||
* Streaming a sound (#349) on the page's side: the copy fetched a piece at a time, its first seconds
|
||||
* decoded ahead, and a `stream-player` kept fed along its plan – every sample where it belongs, at a
|
||||
* jump as well as round a loop, and no further ahead than it needs.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { StreamPlan } from '../../src/core/music/streams';
|
||||
import { StreamFeed, StreamFile } from '../../src/viewport/streaming';
|
||||
import { decoders, fakeCopy, PRE_SKIP, serveCopies, stubDecoder } from './fakeStream';
|
||||
|
||||
const URL = '/sounds/copy.ogg';
|
||||
const settle = () => new Promise((resolve) => setTimeout(resolve, 0));
|
||||
|
||||
/** A worklet node as the feed sees it: what was posted to it, and a way to report what it read. */
|
||||
function fakeNode() {
|
||||
const posted: unknown[] = [];
|
||||
const node = {
|
||||
port: {
|
||||
onmessage: null as ((event: { data: unknown }) => void) | null,
|
||||
postMessage: (data: unknown) => posted.push(data),
|
||||
close: () => {},
|
||||
},
|
||||
disconnect: () => {},
|
||||
};
|
||||
const samples = () =>
|
||||
(posted.filter((data) => Array.isArray(data) && data.length > 0) as Float32Array[][]).flatMap((chunk) =>
|
||||
Array.from(chunk[0]!),
|
||||
);
|
||||
const read = (used: number) => node.port.onmessage?.({ data: { used, skipped: 0 } });
|
||||
return { node: node as unknown as AudioWorkletNode, posted, samples, read };
|
||||
}
|
||||
|
||||
/** A feed as the library makes it: its opening handed over first, then attached to the player. */
|
||||
function feedOf(file: StreamFile, plan: StreamPlan, fake: ReturnType<typeof fakeNode>, ended = () => {}) {
|
||||
const feed = new StreamFeed(file, plan, ended);
|
||||
fake.posted.push(feed.opening());
|
||||
feed.attach(fake.node);
|
||||
return feed;
|
||||
}
|
||||
|
||||
/** Waits until the feed has nothing left to do for now. */
|
||||
async function drain(): Promise<void> {
|
||||
for (let i = 0; i < 400; i++) await settle();
|
||||
}
|
||||
|
||||
beforeEach(() => stubDecoder());
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
|
||||
describe('a copy streamed', () => {
|
||||
it('fetches pieces until its first two seconds are there and decodes them, its pre-skip cut off', async () => {
|
||||
const copy = fakeCopy(10);
|
||||
const server = serveCopies({ [URL]: copy.bytes });
|
||||
const file = new StreamFile(URL, copy.frames);
|
||||
expect(await file.ready()).toBe(true);
|
||||
expect(file.head![0]!.length).toBe(96000);
|
||||
expect([file.head![0]![0], file.head![0]![95999], file.head![1]![0]]).toEqual([
|
||||
PRE_SKIP,
|
||||
PRE_SKIP + 95999,
|
||||
PRE_SKIP + 1e6,
|
||||
]);
|
||||
expect(server.asked[0]).toBe('bytes=0-262143');
|
||||
expect(file.bytes).toBeLessThan(copy.bytes.length);
|
||||
});
|
||||
|
||||
it('takes the whole file from a server that leaves `Range` out', async () => {
|
||||
const copy = fakeCopy(5);
|
||||
const server = serveCopies({ [URL]: copy.bytes }, false);
|
||||
const file = new StreamFile(URL, copy.frames);
|
||||
expect(await file.ready()).toBe(true);
|
||||
expect(server.asked).toHaveLength(1);
|
||||
expect(file.bytes).toBe(copy.bytes.length);
|
||||
});
|
||||
|
||||
it('sends its head at once, then decodes on from there to the sample, no more than four seconds ahead', async () => {
|
||||
const copy = fakeCopy(10);
|
||||
serveCopies({ [URL]: copy.bytes });
|
||||
const file = new StreamFile(URL, copy.frames);
|
||||
await file.ready();
|
||||
const player = fakeNode();
|
||||
const { samples, read } = player;
|
||||
const feed = new StreamFeed(file, { from: 0, until: copy.frames, loop: null }, () => {});
|
||||
// The head goes with the player as it is made; the rest comes by message.
|
||||
expect(feed.opening()[0]).toHaveLength(96000);
|
||||
feed.attach(player.node);
|
||||
expect(samples()).toHaveLength(0);
|
||||
await drain();
|
||||
const sent = samples();
|
||||
expect(sent).toHaveLength(2 * 48000);
|
||||
expect(sent.every((value, i) => value === PRE_SKIP + 96000 + i)).toBe(true);
|
||||
read(48000);
|
||||
await drain();
|
||||
expect(samples()).toHaveLength(3 * 48000);
|
||||
expect(samples()[3 * 48000 - 1]).toBe(PRE_SKIP + 5 * 48000 - 1);
|
||||
});
|
||||
|
||||
it('jumps in mid-way after settling the decoder on what comes before', async () => {
|
||||
const copy = fakeCopy(10);
|
||||
serveCopies({ [URL]: copy.bytes });
|
||||
const file = new StreamFile(URL, copy.frames);
|
||||
await file.ready();
|
||||
const player = fakeNode();
|
||||
const { samples } = player;
|
||||
const from = 6 * 48000 + 123;
|
||||
feedOf(file, { from, until: copy.frames, loop: null }, player);
|
||||
await drain();
|
||||
expect(samples()[0]).toBe(PRE_SKIP + from);
|
||||
expect(samples().every((value, i) => value === PRE_SKIP + from + i)).toBe(true);
|
||||
const fed = decoders.at(-1)!.fed;
|
||||
expect(fed[0]).toBeLessThanOrEqual(PRE_SKIP + from - 32 * 960);
|
||||
expect(fed[0]).toBeGreaterThan(PRE_SKIP + from - 34 * 960);
|
||||
});
|
||||
|
||||
it('goes round its loop to the sample and says when it has played out once', async () => {
|
||||
const copy = fakeCopy(3);
|
||||
serveCopies({ [URL]: copy.bytes });
|
||||
const file = new StreamFile(URL, copy.frames);
|
||||
await file.ready();
|
||||
const looped = fakeNode();
|
||||
const start = 100000;
|
||||
feedOf(file, { from: 0, until: copy.frames, loop: { start, end: copy.frames } }, looped);
|
||||
await drain();
|
||||
const sent = looped.samples();
|
||||
expect(sent[copy.frames - 1]).toBe(PRE_SKIP + copy.frames - 1);
|
||||
expect(sent[copy.frames]).toBe(PRE_SKIP + start);
|
||||
const round = copy.frames - start;
|
||||
expect(sent.length).toBeGreaterThan(copy.frames + round);
|
||||
expect(sent.slice(copy.frames).every((value, i) => value === PRE_SKIP + start + (i % round))).toBe(true);
|
||||
|
||||
const once = fakeNode();
|
||||
let ended = false;
|
||||
feedOf(file, { from: 0, until: copy.frames, loop: null }, once, () => (ended = true));
|
||||
await drain();
|
||||
expect(once.samples()).toHaveLength(copy.frames);
|
||||
expect(once.posted.at(-1)).toBe('end');
|
||||
once.node.port.onmessage?.({ data: 'ended' } as never);
|
||||
expect(ended).toBe(true);
|
||||
});
|
||||
|
||||
it('stops the player and its decoder', async () => {
|
||||
const copy = fakeCopy(10);
|
||||
serveCopies({ [URL]: copy.bytes });
|
||||
const file = new StreamFile(URL, copy.frames);
|
||||
await file.ready();
|
||||
const player = fakeNode();
|
||||
const { posted } = player;
|
||||
const feed = feedOf(file, { from: 5 * 48000, until: copy.frames, loop: null }, player);
|
||||
await drain();
|
||||
feed.stop();
|
||||
expect(posted.at(-1)).toBe('stop');
|
||||
expect(decoders.at(-1)!.closed).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user