Stream long sounds from an Opus copy instead of decoding them (#349)

A sound was decoded whole as soon as the project opened, in the editor
and in a released game: five minutes of stereo held some 115 MB for as
long as the page was open. A sound with Stream on now plays from a copy
as Opus in Ogg, fetched a piece at a time and decoded a few seconds
ahead by WebCodecs, so a game holds a few megabytes for it.

- The Stream switch is a row of the sound, off by default. The Sound
  tab suggests it from a minute on and says how far the copy has got.
- StreamKeeper makes the copies in the browser, a moment after the
  last change, one at a time: the sound's own and one per clip of its
  cut, fades and Reverse baked in. A copy that is no longer wanted is
  taken off and deleted once the project is saved without it.
- streamTable says which played names stream from which copy. A clip
  without a fitting copy, a Sample and a Reverb impulse stay decoded.
  A sound with Stream on is never decoded ahead.
- The page fetches by Range and takes the whole file from a server
  that ignores it, as the CI's PHP server does. It keeps the bytes
  and the first two seconds decoded, the rest goes four seconds ahead.
- A jump decodes 32 packets before the place first. Measured in
  Chrome, what follows is then the same to the bit as decoding from
  the start. With 4 packets it was 0.06 off.
- The stream-player on the FX worklet reads the samples at the rate
  of the context along a Catmull-Rom curve. What comes late is silent
  and skipped once it is there, so a clip keeps its beat.
- The opening stretch goes with the player as it is made. Sent by
  message it came a few milliseconds late, and 557 samples of the
  attack were lost.
- Pitch on a streamed sound is a shifter after the player, as on a
  bus: the length stays and it is heard 42.7 ms late.
- Bakes of patterns and songs (#340) always stream now.
- The song timeline draws a streamed sound from its copy decoded at
  8 kHz, keeping only the peaks.

Tests:
- Gate: CHECK EXIT 0, 280 files, 4312 tests (31 new: ogg +3,
  stream-queue 6, streams 11, streaming 6, sound-library +5).
- e2e 34: ALL PASSED, 106 checks (7 new). Five minutes of mono,
  4.8 MB as WAV, 3.6 MB as its copy: streamed the renderer grows by
  5.8 MB, decoded by 69.6 MB. Offline at 48 kHz, start, loop seam and
  a reversed clip are the copy decoded whole to the sample (0).
- Counter-checks red as they should be: no pre-roll, no catch-up of
  late samples at the end, no opening stretch, a streamed sound
  decoded ahead.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/ogg.test.ts

@@ -3,7 +3,15 @@
  * page says it has got to, and the pre-skip and length a decoder trims by.
  */
 import { describe, expect, it } from 'vitest';
-import { oggCrc, oggOpus, opusHead, preSkipOf, readOggOpus } from '../../src/core/music/ogg';
+import {
+  OggOpusReader,
+  oggCrc,
+  oggOpus,
+  opusHead,
+  opusPacketSamples,
+  preSkipOf,
+  readOggOpus,
+} from '../../src/core/music/ogg';
 
 const packet = (size: number, fill: number) => ({ data: new Uint8Array(size).fill(fill), samples: 960 });
 
@@ -65,3 +73,49 @@ describe('Opus in Ogg', () => {
     expect(preSkipOf(new Uint8Array(19))).toBeNull();
   });
 });
+
+describe('Opus in Ogg as it comes in (#349)', () => {
+  /** A CELT packet of 20 ms at full band: config 31, one frame. */
+  const celt = (size: number, fill: number) => {
+    const data = new Uint8Array(size).fill(fill);
+    data[0] = 31 << 3;
+    return { data, samples: 960 };
+  };
+  const file = oggOpus({
+    channels: 2,
+    preSkip: 312,
+    frames: 300 * 960 - 312,
+    packets: Array.from({ length: 300 }, (_, i) => celt(100 + (i % 300), i % 256)),
+  });
+
+  it('reads a file pushed in pieces as it reads it whole, each packet placed where it starts', () => {
+    const reader = new OggOpusReader();
+    for (let at = 0, size = 1; at < file.length; at += size, size = (size * 7 + 3) % 4000) {
+      reader.push(file.subarray(at, at + size));
+    }
+    const whole = readOggOpus(file);
+    expect(reader.packets.map((p) => p.data)).toEqual(whole.packets);
+    expect(reader.packets.slice(0, 3).map((p) => p.at)).toEqual([0, 960, 1920]);
+    expect(reader.packets.at(-1)!.at).toBe(299 * 960);
+    expect([reader.channels, reader.preSkip, reader.ended]).toEqual([2, 312, true]);
+  });
+
+  it('holds a page back until all of it is there', () => {
+    const reader = new OggOpusReader();
+    reader.push(file.subarray(0, 4000));
+    const seen = reader.packets.length;
+    expect(reader.ended).toBe(false);
+    reader.push(file.subarray(4000));
+    expect(reader.packets.length).toBeGreaterThan(seen);
+    expect(reader.packets).toHaveLength(300);
+  });
+
+  it('counts the samples of a packet from its TOC byte', () => {
+    expect(opusPacketSamples(Uint8Array.of(31 << 3))).toBe(960);
+    expect(opusPacketSamples(Uint8Array.of((28 << 3) | 1))).toBe(240);
+    expect(opusPacketSamples(Uint8Array.of((3 << 3) | 2))).toBe(2 * 2880);
+    expect(opusPacketSamples(Uint8Array.of((13 << 3) | 3, 3))).toBe(3 * 960);
+    expect(opusPacketSamples(Uint8Array.of(12 << 3))).toBe(480);
+    expect(opusPacketSamples(new Uint8Array(0))).toBe(0);
+  });
+});

tests/core/sounds.test.ts

@@ -37,6 +37,8 @@ import {
   loopsOf,
   REVERSE_KEY,
   START_KEY,
+  STREAM_KEY,
+  streamsOf,
   VOLUME_KEY,
   volumeOf,
 } from '../../src/core/properties/sounds';
@@ -129,8 +131,10 @@ describe('the sound asset', () => {
     expect(asset).toMatchObject({ type: 'sound', file: '' });
     expect(volumeOf(asset.props)).toBe(1);
     expect(loopsOf(asset.props)).toBe(false);
+    // Decoded whole unless it is switched to stream (#349).
+    expect(streamsOf(asset.props)).toBe(false);
     // The cut belongs to a clip of the sound (#357), the sound is the whole file.
-    expect(schemaFor('sound').map((def) => def.key)).toEqual([VOLUME_KEY, LOOP_KEY, 'fx']);
+    expect(schemaFor('sound').map((def) => def.key)).toEqual([VOLUME_KEY, LOOP_KEY, STREAM_KEY, 'fx']);
     expect(CLIP_ROWS.map((def) => def.key)).toEqual([
       START_KEY,
       END_KEY,

tests/core/stream-queue.test.ts

@@ -0,0 +1,83 @@
+/**
+ * What a streamed sound (#349) is played from on the audio thread: the samples in order, read at the
+ * rate of the context, silent where they come late and skipped once they are there.
+ */
+import { describe, expect, it } from 'vitest';
+import { STREAM_RATE, StreamQueue } from '../../src/core/music/streamQueue';
+
+const ramp = (from: number, count: number) => Float32Array.from({ length: count }, (_, i) => from + i);
+const block = (size = 128) => [new Float32Array(size), new Float32Array(size)];
+
+describe('the queue of a streamed sound', () => {
+  it('plays what it is sent sample for sample at the rate of the stream, across its chunks', () => {
+    const queue = new StreamQueue(2, 1);
+    queue.push([ramp(0, 100), ramp(1000, 100)]);
+    queue.push([ramp(100, 100), ramp(1100, 100)]);
+    queue.end();
+    const out = block();
+    expect(queue.render(out)).toBe(true);
+    expect(Array.from(out[0]!.subarray(98, 102))).toEqual([98, 99, 100, 101]);
+    expect(Array.from(out[1]!.subarray(0, 2))).toEqual([1000, 1001]);
+    expect(queue.render(out)).toBe(false);
+    expect(Array.from(out[0]!.subarray(70, 74))).toEqual([198, 199, 0, 0]);
+    expect(queue.used).toBe(200);
+  });
+
+  it('plays a mono stream on both sides', () => {
+    const queue = new StreamQueue(1, 1);
+    queue.push([ramp(5, 300)]);
+    const out = block();
+    queue.render(out);
+    expect(out[1]![3]).toBe(8);
+  });
+
+  it('reads it at the rate of a context running at 44.1 kHz along a smooth curve', () => {
+    const step = STREAM_RATE / 44100;
+    const queue = new StreamQueue(1, step);
+    const sine = (i: number) => Math.sin((2 * Math.PI * 440 * i) / STREAM_RATE);
+    for (let at = 0; at < 2 * STREAM_RATE; at += 4800)
+      queue.push([Float32Array.from({ length: 4800 }, (_, i) => sine(at + i))]);
+    const out = [new Float32Array(44100)];
+    queue.render(out);
+    let worst = 0;
+    for (let i = 4; i < 44000; i++) worst = Math.max(worst, Math.abs(out[0]![i]! - sine(i * step)));
+    expect(worst).toBeLessThan(1e-4);
+    expect(queue.used).toBe(Math.floor(44100 * step));
+  });
+
+  it('is silent while what is due has not come, and skips as much once it has, so it keeps time', () => {
+    const queue = new StreamQueue(1, 1);
+    queue.push([ramp(0, 64)]);
+    const out = [new Float32Array(128)];
+    queue.render(out);
+    // The last two it had are held for the curve and come late with the rest.
+    expect(out[0]![61]).toBe(61);
+    expect(out[0]![100]).toBe(0);
+    queue.push([ramp(64, 1000)]);
+    expect(queue.used).toBe(128);
+    expect(queue.skipped).toBe(66);
+    queue.render(out);
+    expect(out[0]![0]).toBe(128);
+  });
+
+  it('says it has played out once it has ended and all of it has been read', () => {
+    const queue = new StreamQueue(2, 1);
+    queue.push([ramp(0, 300), ramp(0, 300)]);
+    queue.end();
+    expect(queue.render(block())).toBe(true);
+    expect(queue.render(block())).toBe(true);
+    const out = block();
+    expect(queue.render(out)).toBe(false);
+    expect(Array.from(out[0]!.subarray(42, 46))).toEqual([298, 299, 0, 0]);
+    expect(queue.done).toBe(true);
+  });
+
+  it('skips what came too late even when nothing more comes after it', () => {
+    const queue = new StreamQueue(1, 1);
+    queue.push([ramp(0, 10)]);
+    queue.render(block());
+    queue.end();
+    expect(queue.done).toBe(true);
+    expect(queue.render(block())).toBe(false);
+  });
+});

tests/core/streams.test.ts

@@ -0,0 +1,180 @@
+/**
+ * Sounds streamed rather than decoded whole (#349): which names a game streams from which copy, what
+ * copies are missing or left over, how a played sound is walked through its copy, and that a copy is
+ * kept with the project.
+ */
+import { describe, expect, it } from 'vitest';
+import {
+  CreateAssetCommand,
+  CreateSoundCommand,
+  SetAssetPropertyCommand,
+} from '../../src/core/commands/assetCommands';
+import { AddClipCommand, SetClipPropsCommand } from '../../src/core/commands/clipCommands';
+import type { SongAsset, SoundAsset, SoundStream } from '../../src/core/model/Asset';
+import { projectFiles } from '../../src/core/model/projectFiles';
+import { cutName } from '../../src/core/model/sound';
+import { STREAM_RATE } from '../../src/core/music/streamQueue';
+import {
+  heldStream,
+  isStreamUpload,
+  playedPlan,
+  StreamCursor,
+  streamTable,
+  streamUploadName,
+  streamWork,
+} from '../../src/core/music/streams';
+import { cutOf } from '../../src/core/properties/sounds';
+import { deserializeProject, serializeProject } from '../../src/core/store/serialize';
+import { makeSpriteScene } from '../helpers';
+
+const FILE = `${'a1'.repeat(6)}/${'b2'.repeat(32)}.wav`;
+const COPY = `${'a1'.repeat(6)}/${'c3'.repeat(32)}.ogg`;
+const CLIP_COPY = `${'a1'.repeat(6)}/${'d4'.repeat(32)}.ogg`;
+
+function streamed() {
+  const f = makeSpriteScene('Box');
+  const made = new CreateSoundCommand('Music', FILE);
+  f.stack.execute(made);
+  const sound = made.asset;
+  const clip = new AddClipCommand(sound.id, { start: 1, end: 3 });
+  f.stack.execute(clip);
+  const whole = new AddClipCommand(sound.id, { start: 0, end: 0 });
+  f.stack.execute(whole);
+  const turn = (on: boolean) => f.stack.execute(new SetAssetPropertyCommand(sound.id, 'stream', on));
+  const copy = (from: string, file: string): SoundStream => ({ file, from, frames: 96000, bytes: 4000 });
+  const cut = () => cutName(FILE, cutOf(sound.clips![0]!.props));
+  return { ...f, sound, clipId: clip.clipId, wholeId: whole.clipId, turn, copy, cut };
+}
+
+describe('which sounds stream', () => {
+  it('streams nothing while the switch is off, whatever copies there are', () => {
+    const f = streamed();
+    f.sound.stream = f.copy(FILE, COPY);
+    expect(streamTable(f.project).size).toBe(0);
+  });
+
+  it('streams a sound from a copy of its file, a clip from a copy of its cut, a whole-file clip as the sound', () => {
+    const f = streamed();
+    f.turn(true);
+    f.sound.stream = f.copy(FILE, COPY);
+    f.sound.clips![0]!.stream = f.copy(f.cut(), CLIP_COPY);
+    expect([...streamTable(f.project)]).toEqual([
+      [FILE, { file: COPY, frames: 96000 }],
+      [f.cut(), { file: CLIP_COPY, frames: 96000 }],
+    ]);
+  });
+
+  it('plays a sound or a clip decoded while its copy is of something else', () => {
+    const f = streamed();
+    f.turn(true);
+    f.sound.stream = f.copy(FILE, COPY);
+    f.sound.clips![0]!.stream = f.copy(f.cut(), CLIP_COPY);
+    f.stack.execute(new SetClipPropsCommand(f.sound.id, f.clipId, { end: 2.5 }));
+    expect([...streamTable(f.project).keys()]).toEqual([FILE]);
+  });
+
+  it('asks for a copy of the sound and of each clip with a cut of its own, and lets them go once it is off', () => {
+    const f = streamed();
+    expect(streamWork(f.project)).toEqual({ jobs: [], drops: [] });
+    f.turn(true);
+    expect(streamWork(f.project).jobs).toEqual([
+      { assetId: f.sound.id, clipId: null, from: FILE },
+      { assetId: f.sound.id, clipId: f.clipId, from: f.cut() },
+    ]);
+    f.sound.stream = f.copy(FILE, COPY);
+    f.sound.clips![0]!.stream = f.copy(f.cut(), CLIP_COPY);
+    expect(streamWork(f.project)).toEqual({ jobs: [], drops: [] });
+    f.stack.execute(new SetClipPropsCommand(f.sound.id, f.clipId, { reverse: true }));
+    expect(streamWork(f.project).jobs).toEqual([{ assetId: f.sound.id, clipId: f.clipId, from: f.cut() }]);
+    f.turn(false);
+    expect(streamWork(f.project)).toEqual({
+      jobs: [],
+      drops: [
+        { assetId: f.sound.id, clipId: null },
+        { assetId: f.sound.id, clipId: f.clipId },
+      ],
+    });
+  });
+
+  it('leaves a sound that streams out of what is decoded ahead, its copy made or not', () => {
+    const f = streamed();
+    expect(projectFiles(f.project).sounds).toEqual([FILE]);
+    f.turn(true);
+    expect(projectFiles(f.project).sounds).toEqual([]);
+  });
+
+  it('streams every bake from itself', () => {
+    const f = streamed();
+    const made = new CreateAssetCommand('song', 'Theme');
+    f.stack.execute(made);
+    (made.asset as SongAsset).bake = {
+      file: COPY,
+      print: 'x',
+      peak: 1,
+      length: 100,
+      loopEnd: 80,
+      head: 100,
+      round: 40,
+      bytes: 9,
+    };
+    expect(streamTable(f.project).get(COPY)).toEqual({ file: COPY, frames: 140 });
+  });
+});
+
+describe('walking a copy', () => {
+  it('plays a sound from where it is asked to, round its whole length where it loops', () => {
+    expect(playedPlan(1000, 0, false)).toEqual({ from: 0, until: 1000, loop: null });
+    expect(playedPlan(STREAM_RATE * 2, 2.5, true)).toEqual({
+      from: STREAM_RATE / 2,
+      until: STREAM_RATE * 2,
+      loop: { start: 0, end: STREAM_RATE * 2 },
+    });
+    expect(playedPlan(1000, 1, false).from).toBe(1000);
+  });
+
+  it('reads straight on to the end of a run, then round the loop, starting a new stretch there', () => {
+    const cursor = new StreamCursor({ from: 50, until: 120, loop: { start: 100, end: 130 } });
+    const read: number[][] = [];
+    for (let i = 0; i < 5; i++) {
+      const span = cursor.next(40)!;
+      read.push([span.from, span.to]);
+    }
+    expect(read).toEqual([
+      [50, 90],
+      [90, 120],
+      [100, 130],
+      [100, 130],
+      [100, 130],
+    ]);
+    expect(cursor.peek()).toBe(100);
+  });
+
+  it('ends a run without a loop', () => {
+    const cursor = new StreamCursor({ from: 0, until: 10, loop: null });
+    expect(cursor.next(100)).toEqual({ from: 0, to: 10 });
+    expect(cursor.peek()).toBeNull();
+    expect(cursor.next(100)).toBeNull();
+  });
+});
+
+describe('a copy kept with the project', () => {
+  it('comes back from a save, and one that is not right is dropped', () => {
+    const f = streamed();
+    f.sound.stream = f.copy(FILE, COPY);
+    f.sound.clips![0]!.stream = f.copy(f.cut(), CLIP_COPY);
+    const back = deserializeProject(serializeProject(f.project));
+    const sound = back.assets.find((a) => a.id === f.sound.id) as SoundAsset;
+    expect(sound.stream).toEqual(f.copy(FILE, COPY));
+    expect(sound.clips![0]!.stream).toEqual(f.copy(f.cut(), CLIP_COPY));
+    expect(sound.clips![1]!.stream).toBeUndefined();
+    expect(heldStream({ ...f.copy(FILE, COPY), frames: 1.5 })).toBeNull();
+    expect(heldStream({ ...f.copy('', COPY) })).toBeNull();
+    expect(heldStream({ ...f.copy(FILE, 'notes.txt') })).toBeNull();
+  });
+
+  it('is uploaded under a mark the sound picker leaves out by', () => {
+    expect(streamUploadName('Music / Intro')).toBe('[stream] Music - Intro.ogg');
+    expect(isStreamUpload(streamUploadName('Music'))).toBe(true);
+    expect(isStreamUpload('Music.ogg')).toBe(false);
+  });
+});