A sound was decoded whole as soon as the project opened, in the editor and in a released game: five minutes of stereo held some 115 MB for as long as the page was open. A sound with Stream on now plays from a copy as Opus in Ogg, fetched a piece at a time and decoded a few seconds ahead by WebCodecs, so a game holds a few megabytes for it. - The Stream switch is a row of the sound, off by default. The Sound tab suggests it from a minute on and says how far the copy has got. - StreamKeeper makes the copies in the browser, a moment after the last change, one at a time: the sound's own and one per clip of its cut, fades and Reverse baked in. A copy that is no longer wanted is taken off and deleted once the project is saved without it. - streamTable says which played names stream from which copy. A clip without a fitting copy, a Sample and a Reverb impulse stay decoded. A sound with Stream on is never decoded ahead. - The page fetches by Range and takes the whole file from a server that ignores it, as the CI's PHP server does. It keeps the bytes and the first two seconds decoded, the rest goes four seconds ahead. - A jump decodes 32 packets before the place first. Measured in Chrome, what follows is then the same to the bit as decoding from the start. With 4 packets it was 0.06 off. - The stream-player on the FX worklet reads the samples at the rate of the context along a Catmull-Rom curve. What comes late is silent and skipped once it is there, so a clip keeps its beat. - The opening stretch goes with the player as it is made. Sent by message it came a few milliseconds late, and 557 samples of the attack were lost. - Pitch on a streamed sound is a shifter after the player, as on a bus: the length stays and it is heard 42.7 ms late. - Bakes of patterns and songs (#340) always stream now. - The song timeline draws a streamed sound from its copy decoded at 8 kHz, keeping only the peaks. Tests: - Gate: CHECK EXIT 0, 280 files, 4312 tests (31 new: ogg +3, stream-queue 6, streams 11, streaming 6, sound-library +5). - e2e 34: ALL PASSED, 106 checks (7 new). Five minutes of mono, 4.8 MB as WAV, 3.6 MB as its copy: streamed the renderer grows by 5.8 MB, decoded by 69.6 MB. Offline at 48 kHz, start, loop seam and a reversed clip are the copy decoded whole to the sample (0). - Counter-checks red as they should be: no pre-roll, no catch-up of late samples at the end, no opening stretch, a streamed sound decoded ahead. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
181 lines
6.9 KiB
TypeScript
181 lines
6.9 KiB
TypeScript
/**
|
|
* Sounds streamed rather than decoded whole (#349): which names a game streams from which copy, what
|
|
* copies are missing or left over, how a played sound is walked through its copy, and that a copy is
|
|
* kept with the project.
|
|
*/
|
|
import { describe, expect, it } from 'vitest';
|
|
import {
|
|
CreateAssetCommand,
|
|
CreateSoundCommand,
|
|
SetAssetPropertyCommand,
|
|
} from '../../src/core/commands/assetCommands';
|
|
import { AddClipCommand, SetClipPropsCommand } from '../../src/core/commands/clipCommands';
|
|
import type { SongAsset, SoundAsset, SoundStream } from '../../src/core/model/Asset';
|
|
import { projectFiles } from '../../src/core/model/projectFiles';
|
|
import { cutName } from '../../src/core/model/sound';
|
|
import { STREAM_RATE } from '../../src/core/music/streamQueue';
|
|
import {
|
|
heldStream,
|
|
isStreamUpload,
|
|
playedPlan,
|
|
StreamCursor,
|
|
streamTable,
|
|
streamUploadName,
|
|
streamWork,
|
|
} from '../../src/core/music/streams';
|
|
import { cutOf } from '../../src/core/properties/sounds';
|
|
import { deserializeProject, serializeProject } from '../../src/core/store/serialize';
|
|
import { makeSpriteScene } from '../helpers';
|
|
|
|
const FILE = `${'a1'.repeat(6)}/${'b2'.repeat(32)}.wav`;
|
|
const COPY = `${'a1'.repeat(6)}/${'c3'.repeat(32)}.ogg`;
|
|
const CLIP_COPY = `${'a1'.repeat(6)}/${'d4'.repeat(32)}.ogg`;
|
|
|
|
function streamed() {
|
|
const f = makeSpriteScene('Box');
|
|
const made = new CreateSoundCommand('Music', FILE);
|
|
f.stack.execute(made);
|
|
const sound = made.asset;
|
|
const clip = new AddClipCommand(sound.id, { start: 1, end: 3 });
|
|
f.stack.execute(clip);
|
|
const whole = new AddClipCommand(sound.id, { start: 0, end: 0 });
|
|
f.stack.execute(whole);
|
|
const turn = (on: boolean) => f.stack.execute(new SetAssetPropertyCommand(sound.id, 'stream', on));
|
|
const copy = (from: string, file: string): SoundStream => ({ file, from, frames: 96000, bytes: 4000 });
|
|
const cut = () => cutName(FILE, cutOf(sound.clips![0]!.props));
|
|
return { ...f, sound, clipId: clip.clipId, wholeId: whole.clipId, turn, copy, cut };
|
|
}
|
|
|
|
describe('which sounds stream', () => {
|
|
it('streams nothing while the switch is off, whatever copies there are', () => {
|
|
const f = streamed();
|
|
f.sound.stream = f.copy(FILE, COPY);
|
|
expect(streamTable(f.project).size).toBe(0);
|
|
});
|
|
|
|
it('streams a sound from a copy of its file, a clip from a copy of its cut, a whole-file clip as the sound', () => {
|
|
const f = streamed();
|
|
f.turn(true);
|
|
f.sound.stream = f.copy(FILE, COPY);
|
|
f.sound.clips![0]!.stream = f.copy(f.cut(), CLIP_COPY);
|
|
expect([...streamTable(f.project)]).toEqual([
|
|
[FILE, { file: COPY, frames: 96000 }],
|
|
[f.cut(), { file: CLIP_COPY, frames: 96000 }],
|
|
]);
|
|
});
|
|
|
|
it('plays a sound or a clip decoded while its copy is of something else', () => {
|
|
const f = streamed();
|
|
f.turn(true);
|
|
f.sound.stream = f.copy(FILE, COPY);
|
|
f.sound.clips![0]!.stream = f.copy(f.cut(), CLIP_COPY);
|
|
f.stack.execute(new SetClipPropsCommand(f.sound.id, f.clipId, { end: 2.5 }));
|
|
expect([...streamTable(f.project).keys()]).toEqual([FILE]);
|
|
});
|
|
|
|
it('asks for a copy of the sound and of each clip with a cut of its own, and lets them go once it is off', () => {
|
|
const f = streamed();
|
|
expect(streamWork(f.project)).toEqual({ jobs: [], drops: [] });
|
|
f.turn(true);
|
|
expect(streamWork(f.project).jobs).toEqual([
|
|
{ assetId: f.sound.id, clipId: null, from: FILE },
|
|
{ assetId: f.sound.id, clipId: f.clipId, from: f.cut() },
|
|
]);
|
|
f.sound.stream = f.copy(FILE, COPY);
|
|
f.sound.clips![0]!.stream = f.copy(f.cut(), CLIP_COPY);
|
|
expect(streamWork(f.project)).toEqual({ jobs: [], drops: [] });
|
|
f.stack.execute(new SetClipPropsCommand(f.sound.id, f.clipId, { reverse: true }));
|
|
expect(streamWork(f.project).jobs).toEqual([{ assetId: f.sound.id, clipId: f.clipId, from: f.cut() }]);
|
|
f.turn(false);
|
|
expect(streamWork(f.project)).toEqual({
|
|
jobs: [],
|
|
drops: [
|
|
{ assetId: f.sound.id, clipId: null },
|
|
{ assetId: f.sound.id, clipId: f.clipId },
|
|
],
|
|
});
|
|
});
|
|
|
|
it('leaves a sound that streams out of what is decoded ahead, its copy made or not', () => {
|
|
const f = streamed();
|
|
expect(projectFiles(f.project).sounds).toEqual([FILE]);
|
|
f.turn(true);
|
|
expect(projectFiles(f.project).sounds).toEqual([]);
|
|
});
|
|
|
|
it('streams every bake from itself', () => {
|
|
const f = streamed();
|
|
const made = new CreateAssetCommand('song', 'Theme');
|
|
f.stack.execute(made);
|
|
(made.asset as SongAsset).bake = {
|
|
file: COPY,
|
|
print: 'x',
|
|
peak: 1,
|
|
length: 100,
|
|
loopEnd: 80,
|
|
head: 100,
|
|
round: 40,
|
|
bytes: 9,
|
|
};
|
|
expect(streamTable(f.project).get(COPY)).toEqual({ file: COPY, frames: 140 });
|
|
});
|
|
});
|
|
|
|
describe('walking a copy', () => {
|
|
it('plays a sound from where it is asked to, round its whole length where it loops', () => {
|
|
expect(playedPlan(1000, 0, false)).toEqual({ from: 0, until: 1000, loop: null });
|
|
expect(playedPlan(STREAM_RATE * 2, 2.5, true)).toEqual({
|
|
from: STREAM_RATE / 2,
|
|
until: STREAM_RATE * 2,
|
|
loop: { start: 0, end: STREAM_RATE * 2 },
|
|
});
|
|
expect(playedPlan(1000, 1, false).from).toBe(1000);
|
|
});
|
|
|
|
it('reads straight on to the end of a run, then round the loop, starting a new stretch there', () => {
|
|
const cursor = new StreamCursor({ from: 50, until: 120, loop: { start: 100, end: 130 } });
|
|
const read: number[][] = [];
|
|
for (let i = 0; i < 5; i++) {
|
|
const span = cursor.next(40)!;
|
|
read.push([span.from, span.to]);
|
|
}
|
|
expect(read).toEqual([
|
|
[50, 90],
|
|
[90, 120],
|
|
[100, 130],
|
|
[100, 130],
|
|
[100, 130],
|
|
]);
|
|
expect(cursor.peek()).toBe(100);
|
|
});
|
|
|
|
it('ends a run without a loop', () => {
|
|
const cursor = new StreamCursor({ from: 0, until: 10, loop: null });
|
|
expect(cursor.next(100)).toEqual({ from: 0, to: 10 });
|
|
expect(cursor.peek()).toBeNull();
|
|
expect(cursor.next(100)).toBeNull();
|
|
});
|
|
});
|
|
|
|
describe('a copy kept with the project', () => {
|
|
it('comes back from a save, and one that is not right is dropped', () => {
|
|
const f = streamed();
|
|
f.sound.stream = f.copy(FILE, COPY);
|
|
f.sound.clips![0]!.stream = f.copy(f.cut(), CLIP_COPY);
|
|
const back = deserializeProject(serializeProject(f.project));
|
|
const sound = back.assets.find((a) => a.id === f.sound.id) as SoundAsset;
|
|
expect(sound.stream).toEqual(f.copy(FILE, COPY));
|
|
expect(sound.clips![0]!.stream).toEqual(f.copy(f.cut(), CLIP_COPY));
|
|
expect(sound.clips![1]!.stream).toBeUndefined();
|
|
expect(heldStream({ ...f.copy(FILE, COPY), frames: 1.5 })).toBeNull();
|
|
expect(heldStream({ ...f.copy('', COPY) })).toBeNull();
|
|
expect(heldStream({ ...f.copy(FILE, 'notes.txt') })).toBeNull();
|
|
});
|
|
|
|
it('is uploaded under a mark the sound picker leaves out by', () => {
|
|
expect(streamUploadName('Music / Intro')).toBe('[stream] Music - Intro.ogg');
|
|
expect(isStreamUpload(streamUploadName('Music'))).toBe(true);
|
|
expect(isStreamUpload('Music.ogg')).toBe(false);
|
|
});
|
|
});
|