meleki/feuerwerk
1
0 Fork 0
Code Issues 38 Actions Projects 1 Wiki Activity

Stream long sounds from an Opus copy instead of decoding them (#349)

Browse Source 
A sound was decoded whole as soon as the project opened, in the editor
and in a released game: five minutes of stereo held some 115 MB for as
long as the page was open. A sound with Stream on now plays from a copy
as Opus in Ogg, fetched a piece at a time and decoded a few seconds
ahead by WebCodecs, so a game holds a few megabytes for it.

- The Stream switch is a row of the sound, off by default. The Sound
  tab suggests it from a minute on and says how far the copy has got.
- StreamKeeper makes the copies in the browser, a moment after the
  last change, one at a time: the sound's own and one per clip of its
  cut, fades and Reverse baked in. A copy that is no longer wanted is
  taken off and deleted once the project is saved without it.
- streamTable says which played names stream from which copy. A clip
  without a fitting copy, a Sample and a Reverb impulse stay decoded.
  A sound with Stream on is never decoded ahead.
- The page fetches by Range and takes the whole file from a server
  that ignores it, as the CI's PHP server does. It keeps the bytes
  and the first two seconds decoded, the rest goes four seconds ahead.
- A jump decodes 32 packets before the place first. Measured in
  Chrome, what follows is then the same to the bit as decoding from
  the start. With 4 packets it was 0.06 off.
- The stream-player on the FX worklet reads the samples at the rate
  of the context along a Catmull-Rom curve. What comes late is silent
  and skipped once it is there, so a clip keeps its beat.
- The opening stretch goes with the player as it is made. Sent by
  message it came a few milliseconds late, and 557 samples of the
  attack were lost.
- Pitch on a streamed sound is a shifter after the player, as on a
  bus: the length stays and it is heard 42.7 ms late.
- Bakes of patterns and songs (#340) always stream now.
- The song timeline draws a streamed sound from its copy decoded at
  8 kHz, keeping only the peaks.

Tests:
- Gate: CHECK EXIT 0, 280 files, 4312 tests (31 new: ogg +3,
  stream-queue 6, streams 11, streaming 6, sound-library +5).
- e2e 34: ALL PASSED, 106 checks (7 new). Five minutes of mono,
  4.8 MB as WAV, 3.6 MB as its copy: streamed the renderer grows by
  5.8 MB, decoded by 69.6 MB. Offline at 48 kHz, start, loop seam and
  a reversed clip are the copy decoded whole to the sample (0).
- Counter-checks red as they should be: no pre-roll, no catch-up of
  late samples at the end, no opening stretch, a streamed sound
  decoded ahead.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
adminandClaude Opus 5.5 committed 2026-10-09 20:27:32 +02:00
1 parent 84cd36f902
commit 2764919bb8
30 files changed
+2443 -146
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/core/model/Asset.ts
+17
View File
@@ -317,6 +317,21 @@ export interface SoundAsset extends AssetBase {
file: string;
/** Absent for a sound without clips. */
clips?: SoundClip[];
/** The copy it is streamed from while its `stream` is on (#349). */
stream?: SoundStream;
}
/**
* A sound or a clip of it as Opus in Ogg (#349), what it is streamed from rather than decoded whole.
* Made in the browser from what it plays; `from` says what that was – the file, or the name of the
* cut a clip is – and once it is something else the copy is stale and made again.
*/
export interface SoundStream {
file: string;
from: string;
/** Samples per channel at 48 kHz, and the size of the file, to show. */
frames: number;
bytes: number;
}
/**
@@ -329,6 +344,8 @@ export interface SoundClip {
id: string;
name: string;
props: PropertyBag;
/** Its cut baked to a copy of its own, while its sound streams (#349). */
stream?: SoundStream;
}
/**
src/core/model/projectFiles.ts
+5 -2
View File
@@ -1,4 +1,5 @@
import { validBake } from '../music/bake';
import { streamsOf } from '../properties/sounds';
import { isFontFile } from './font';
import { celFilesOf } from './pixelart';
import type { Project } from './Project';
@@ -17,7 +18,8 @@ export interface ProjectFiles {
* Every file an asset names, used or not: which frame an animation or a tile reaches next is only
* known while it plays, and a file fetched then shows as a gap. A name that is no stored file –
* an upload that failed – is left out, as the libraries would never get it anyway. So is a bake that
* no longer holds (#340): it would never be played, and decoded it weighs as much as the music.
* no longer holds (#340): it would never be played. So is a sound that streams (#349): the sound
* library fetches the start of its copy instead, and decoded ahead it would weigh what streaming saves.
*/
export function projectFiles(project: Project): ProjectFiles {
const files: ProjectFiles = { textures: [], fonts: [], cels: [], sounds: [] };
@@ -25,7 +27,8 @@ export function projectFiles(project: Project): ProjectFiles {
if (asset.type === 'texture' && isTextureFile(asset.file)) files.textures.push(asset.file);
else if (asset.type === 'font' && isFontFile(asset.file)) files.fonts.push(asset.file);
else if (asset.type === 'pixelart') files.cels.push(...celFilesOf(asset));
else if (asset.type === 'sound' && isSoundFile(asset.file)) files.sounds.push(asset.file);
else if (asset.type === 'sound' && isSoundFile(asset.file) && !streamsOf(asset.props))
files.sounds.push(asset.file);
else if ((asset.type === 'pattern' || asset.type === 'song') && validBake(project, asset))
files.sounds.push(asset.bake!.file);
}
src/core/music/ogg.ts
+98 -25
View File
@@ -140,38 +140,111 @@ export interface ReadOpus {
pages: number;
}
/** Reads a file `oggOpus` wrote, checking every page; throws at the first that is not right. */
/** Reads a whole file `oggOpus` wrote, checking every page; throws at the first that is not right. */
export function readOggOpus(bytes: Uint8Array): ReadOpus {
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
const all: Uint8Array[] = [];
let open: Uint8Array[] = [];
let granule = 0;
let pages = 0;
for (let at = 0; at < bytes.length; pages++) {
if (text(bytes.subarray(at, at + 4)) !== 'OggS') throw new Error(`no page at byte ${at}`);
const count = bytes[at + 26]!;
const lacing = bytes.subarray(at + 27, at + 27 + count);
const size = 27 + count + lacing.reduce((sum, value) => sum + value, 0);
const page = bytes.slice(at, at + size);
const crc = view.getUint32(at + 22, true);
page.fill(0, 22, 26);
if (oggCrc(page) !== crc) throw new Error(`page ${pages} has a wrong checksum`);
granule = view.getUint32(at + 6, true) + view.getUint32(at + 10, true) * 2 ** 32;
let from = at + 27 + count;
const reader = new OggOpusReader();
reader.push(bytes);
if (reader.channels === null || reader.preSkip === null) throw new Error('no OpusHead');
return {
channels: reader.channels,
preSkip: reader.preSkip,
granule: reader.granule,
packets: reader.packets.map((packet) => packet.data),
pages: reader.pages,
};
}
/** A packet of a stream (#349) and where it lies, in samples counted from the first, pre-skip and all. */
export interface PlacedPacket {
data: Uint8Array;
at: number;
samples: number;
}
/**
* Reads Ogg Opus as its bytes come in (#349), a page once all of it is there, checking each; throws
* at the first that is not right. The packets after the tags are placed by the samples each stands for.
*/
export class OggOpusReader {
channels: number | null = null;
preSkip: number | null = null;
/** The granule position of the last page read, and whether it was the last of the stream. */
granule = 0;
ended = false;
pages = 0;
readonly packets: PlacedPacket[] = [];
private held: Uint8Array = new Uint8Array(0);
private open: Uint8Array[] = [];
private headers = 0;
private next = 0;
push(bytes: Uint8Array): void {
let data = this.held.length > 0 ? concat([this.held, bytes]) : bytes;
let at = 0;
while (data.length - at >= 27) {
if (text(data.subarray(at, at + 4)) !== 'OggS') throw new Error(`no page at byte ${at}`);
const count = data[at + 26]!;
if (data.length - at < 27 + count) break;
const lacing = data.subarray(at + 27, at + 27 + count);
const size = 27 + count + lacing.reduce((sum, value) => sum + value, 0);
if (data.length - at < size) break;
this.page(data.subarray(at, at + size), lacing);
at += size;
}
data = data.subarray(at);
this.held = data.length > 0 ? data.slice() : new Uint8Array(0);
}
private page(page: Uint8Array, lacing: Uint8Array): void {
const view = new DataView(page.buffer, page.byteOffset, page.byteLength);
const crc = view.getUint32(22, true);
const blank = page.slice();
blank.fill(0, 22, 26);
if (oggCrc(blank) !== crc) throw new Error(`page ${this.pages} has a wrong checksum`);
this.granule = view.getUint32(6, true) + view.getUint32(10, true) * 2 ** 32;
this.ended = (page[5]! & 0x04) !== 0;
this.pages += 1;
let from = 27 + lacing.length;
for (const value of lacing) {
open.push(bytes.subarray(from, from + value));
this.open.push(page.subarray(from, from + value));
from += value;
if (value < 255) {
all.push(concat(open));
open = [];
this.packet(concat(this.open));
this.open = [];
}
}
at += size;
}
const head = all[0];
const preSkip = head ? preSkipOf(head) : null;
if (!head || preSkip === null) throw new Error('no OpusHead');
return { channels: head[9]!, preSkip, granule, packets: all.slice(2), pages };
private packet(data: Uint8Array): void {
if (this.headers === 0) {
this.preSkip = preSkipOf(data);
if (this.preSkip === null) throw new Error('no OpusHead');
this.channels = data[9]!;
}
if (this.headers < 2) {
this.headers += 1;
return;
}
const samples = opusPacketSamples(data);
this.packets.push({ data, at: this.next, samples });
this.next += samples;
}
}
/** The samples per channel at 48 kHz an Opus packet stands for, from its TOC byte (RFC 6716 3.1). */
export function opusPacketSamples(packet: Uint8Array): number {
if (packet.length === 0) return 0;
const toc = packet[0]!;
const config = toc >> 3;
const size =
config < 12
? [480, 960, 1920, 2880][config & 3]!
: config < 16
? [480, 960][config & 1]!
: [120, 240, 480, 960][config & 3]!;
const code = toc & 3;
const frames = code === 0 ? 1 : code === 3 ? (packet[1] ?? 0) & 63 : 2;
return size * frames;
}
function ascii(value: string): Uint8Array {
src/core/music/streamQueue.ts
+144
View File
@@ -0,0 +1,144 @@
/**
* What a streamed sound (#349) is played from on the audio thread: the samples the page decoded, in
* the order they are heard, at `STREAM_RATE`. It reads them out at the rate of the context, `step`
* of its samples per sample heard, between two of them along a cubic curve. Time runs on whatever
* has arrived: a stretch that comes late is silent where it would have been heard and skipped once it
* is there, so a sound in a song stays on its beat.
*/
export const STREAM_RATE = 48000;
export class StreamQueue {
private readonly chunks: Float32Array[][] = [];
/** Where in the first chunk the next sample is, and how far between it and the one after. */
private index = 0;
private fraction = 0;
/** The sample of each channel read last, for the curve. */
private readonly last: number[];
private finished = false;
/** Samples of the stream read so far, heard or skipped, and those that were due but not there. */
used = 0;
owed = 0;
/** Samples that came too late to be heard (test hook). */
skipped = 0;
constructor(
readonly channels: number,
private readonly step: number,
) {
this.last = Array.from({ length: channels }, () => 0);
}
push(chunk: Float32Array[]): void {
if (chunk.length === 0 || (chunk[0]?.length ?? 0) === 0) return;
this.chunks.push(chunk);
this.payOwed();
}
/** Nothing more comes: it ends once what is held has been read. */
end(): void {
this.finished = true;
this.payOwed();
}
/** Samples held and not yet read. */
get held(): number {
let count = -this.index;
for (const chunk of this.chunks) count += chunk[0]!.length;
return Math.max(0, count);
}
/** Whether all of it has been read. */
get done(): boolean {
return this.finished && this.held === 0;
}
/**
* Fills `out` from what is held; what is not there yet is silence. Says whether it goes on: false
* once it has ended and been read to the end.
*/
render(out: Float32Array[]): boolean {
const length = out[0]?.length ?? 0;
for (let i = 0; i < length; i++) {
if (!this.ahead(2)) {
if (this.finished && this.held === 0) {
for (const channel of out) channel.fill(0, i);
return false;
}
if (!this.finished) {
for (const channel of out) channel[i] = 0;
this.owe();
continue;
}
}
const t = this.fraction;
for (let c = 0; c < out.length; c++) {
const side = Math.min(c, this.channels - 1);
out[c]![i] = cubic(this.last[side]!, this.at(side, 0), this.at(side, 1), this.at(side, 2), t);
}
this.advance(this.step);
}
return !this.done;
}
/** Whether `count` samples past the next one are held, or the stream ends before them. */
private ahead(count: number): boolean {
if (this.finished) return this.held > 0;
return this.held > count;
}
/** The sample `k` places after the next one, 0 past the end. */
private at(side: number, k: number): number {
let offset = this.index + k;
for (const chunk of this.chunks) {
const data = chunk[Math.min(side, chunk.length - 1)]!;
if (offset < data.length) return data[offset]!;
offset -= data.length;
}
return 0;
}
private advance(by: number): void {
this.fraction += by;
while (this.fraction >= 1) {
this.fraction -= 1;
this.take();
}
}
/** Moves past the next sample, keeping it for the curve. */
private take(): void {
for (let side = 0; side < this.channels; side++) this.last[side] = this.at(side, 0);
this.index += 1;
this.used += 1;
while (this.chunks.length > 0 && this.index >= this.chunks[0]![0]!.length) {
this.index -= this.chunks[0]![0]!.length;
this.chunks.shift();
}
}
/** A sample was due and not there: skipped once it is. */
private owe(): void {
this.fraction += this.step;
while (this.fraction >= 1) {
this.fraction -= 1;
this.owed += 1;
}
}
private payOwed(): void {
while (this.owed > 0 && this.held > 0) {
this.take();
this.owed -= 1;
this.skipped += 1;
}
}
}
/** Catmull-Rom between `p1` and `p2`, `t` of the way; at 0 it is `p1` to the bit. */
function cubic(p0: number, p1: number, p2: number, p3: number, t: number): number {
if (t === 0) return p1;
const a = -0.5 * p0 + 1.5 * p1 - 1.5 * p2 + 0.5 * p3;
const b = p0 - 2.5 * p1 + 2 * p2 - 0.5 * p3;
const c = -0.5 * p0 + 0.5 * p2;
return ((a * t + b) * t + c) * t + p1;
}
src/core/music/streams.ts
+154
View File
@@ -0,0 +1,154 @@
/**
* Sounds streamed rather than decoded whole (#349). A sound with `stream` on is played from a copy as
* Opus in Ogg, made in the browser, and each of its clips from a copy of its own cut, so no clip needs
* the whole file either. A bake (#340) is Opus already and always streams. What a game plays is still
* named as before – the file, or the cut a clip is – and the table here says which of those names to
* stream from which copy.
*/
import type { SoundAsset, SoundClip, SoundStream } from '../model/Asset';
import type { Project } from '../model/Project';
import { clipsOf, cutName, isSoundFile } from '../model/sound';
import { cutOf, streamsOf } from '../properties/sounds';
import { STREAM_RATE } from './streamQueue';
/** Seconds from which the Sound tab suggests streaming. */
export const STREAM_SUGGESTED = 60;
/** What a name is streamed from: the copy, and how many samples per channel it holds at `STREAM_RATE`. */
export interface StreamedFile {
file: string;
frames: number;
}
/** What a sound or a clip played streamed is read as (#349), in samples of the copy at `STREAM_RATE`. */
export interface StreamPlan {
/** Where it starts. */
from: number;
/** Where its first run ends: the end of the copy, or of the head of a bake. */
until: number;
/** What it runs round after that; null to end there. */
loop: { start: number; end: number } | null;
}
/** A sound or a clip played from `offset` seconds in, round and round where it loops. */
export function playedPlan(frames: number, offset: number, loop: boolean): StreamPlan {
const at = Math.max(0, Math.round(offset * STREAM_RATE));
const from = loop && frames > 0 ? at % frames : Math.min(at, frames);
return { from, until: frames, loop: loop && frames > 0 ? { start: 0, end: frames } : null };
}
/** Walks a plan in stretches of the copy, each read straight through; a loop starts a new one. */
export class StreamCursor {
private at: number;
private end: number;
constructor(private readonly plan: StreamPlan) {
this.at = plan.from;
this.end = plan.until;
}
/** Where the next stretch starts; null once all of it has been read. */
peek(): number | null {
if (this.at < this.end) return this.at;
const loop = this.plan.loop;
return loop && loop.end > loop.start ? loop.start : null;
}
/** The next stretch, at most `most` samples; null once all of it has been read. */
next(most: number): { from: number; to: number } | null {
if (this.at >= this.end) {
const loop = this.plan.loop;
if (!loop || loop.end <= loop.start) return null;
this.at = loop.start;
this.end = loop.end;
}
const from = this.at;
this.at = Math.min(this.end, from + Math.max(1, most));
return { from, to: this.at };
}
}
/**
* Every name a game streams, with its copy: each sound with `stream` on whose copy is of its file,
* each clip of one whose copy is of its cut – a clip without one plays decoded until it is there – and
* every bake, which streams from itself.
*/
export function streamTable(project: Project): Map<string, StreamedFile> {
const table = new Map<string, StreamedFile>();
for (const asset of project.assets) {
if (asset.type === 'sound') {
if (!streamsOf(asset.props) || !asset.file) continue;
const own = heldCopy(asset.stream, asset.file);
if (own) table.set(asset.file, own);
for (const clip of clipsOf(asset)) {
const played = cutName(asset.file, cutOf(clip.props));
const copy = played === asset.file ? own : heldCopy(clip.stream, played);
if (copy) table.set(played, copy);
}
} else if ((asset.type === 'pattern' || asset.type === 'song') && asset.bake) {
const bake = asset.bake;
table.set(bake.file, { file: bake.file, frames: bake.head + bake.round });
}
}
return table;
}
function heldCopy(stream: SoundStream | undefined, played: string): StreamedFile | null {
return stream && stream.from === played ? { file: stream.file, frames: stream.frames } : null;
}
/** A copy to make: of a sound, or of one of its clips, from what it plays now. */
export interface StreamJob {
assetId: string;
clipId: string | null;
from: string;
}
/** A copy that is to go: its sound no longer streams, or a clip of it has become the whole file. */
export interface StreamDrop {
assetId: string;
clipId: string | null;
}
/** What is missing and what is left over, sounds in their order, each sound before its clips. */
export function streamWork(project: Project): { jobs: StreamJob[]; drops: StreamDrop[] } {
const jobs: StreamJob[] = [];
const drops: StreamDrop[] = [];
for (const asset of project.assets) {
if (asset.type !== 'sound') continue;
const on = streamsOf(asset.props) && asset.file !== '';
const keep = (owner: SoundAsset | SoundClip, clipId: string | null, from: string | null) => {
if (from === null) {
if (owner.stream) drops.push({ assetId: asset.id, clipId });
} else if (owner.stream?.from !== from) jobs.push({ assetId: asset.id, clipId, from });
};
keep(asset, null, on ? asset.file : null);
for (const clip of clipsOf(asset)) {
const played = cutName(asset.file, cutOf(clip.props));
keep(clip, clip.id, on && played !== asset.file ? played : null);
}
}
return { jobs, drops };
}
/** Held to what a copy may be, else none. */
export function heldStream(raw: unknown): SoundStream | null {
if (typeof raw !== 'object' || raw === null) return null;
const s = raw as Record<string, unknown>;
const whole = (value: unknown) => typeof value === 'number' && Number.isInteger(value) && value >= 0;
if (typeof s.file !== 'string' || !isSoundFile(s.file) || typeof s.from !== 'string' || s.from === '')
return null;
if (!whole(s.frames) || !whole(s.bytes)) return null;
return { file: s.file, from: s.from, frames: s.frames as number, bytes: s.bytes as number };
}
/** What a copy is uploaded as: like a bake's, behind a mark the sound picker leaves out by. */
const STREAM_MARK = '[stream] ';
export function streamUploadName(name: string): string {
return `${STREAM_MARK}${name.replace(/[/\\]/g, '-')}.ogg`;
}
export function isStreamUpload(name: string): boolean {
return name.startsWith(STREAM_MARK);
}
src/core/properties/PropertySchema.ts
+7 -1
View File
@@ -103,7 +103,11 @@ import {
} from './particles';
import { SHADER_KEY, shaderOf, shaderParamDefs } from './shaders';
import { SHADER_RESOLUTION_KEY, SHADER_RESOLUTION_OPTIONS } from './cameraShader';
import { LOOP_KEY as SOUND_LOOP_KEY, VOLUME_KEY as SOUND_VOLUME_KEY } from './sounds';
import {
LOOP_KEY as SOUND_LOOP_KEY,
STREAM_KEY as SOUND_STREAM_KEY,
VOLUME_KEY as SOUND_VOLUME_KEY,
} from './sounds';
import {
DEFAULT_BARS,
DEFAULT_GRID,
@@ -692,6 +696,8 @@ const ASSET_SCHEMAS: Record<AssetType, PropertySchema> = {
sound: [
{ key: SOUND_VOLUME_KEY, label: 'Volume', kind: 'number', default: 1, min: 0, max: 1, step: 0.05 },
{ key: SOUND_LOOP_KEY, label: 'Loop', kind: 'boolean', default: false },
// A long one is streamed from a copy as Opus rather than decoded whole (#349).
{ key: SOUND_STREAM_KEY, label: 'Stream', kind: 'boolean', default: false },
FX_ROW,
],
// The effects are the asset's graph (see SoundFxAsset), not a property.
src/core/properties/sounds.ts
+6
View File
@@ -7,6 +7,8 @@ import type { PropertyDef } from './PropertySchema';
export const VOLUME_KEY = 'volume';
/** Whether this sound is music: it repeats until something stops it. */
export const LOOP_KEY = 'loop';
/** Whether a game streams it from a copy as Opus rather than decoding all of it first (#349). */
export const STREAM_KEY = 'stream';
/** Where in its file a clip starts and ends, in seconds; an end of 0 is the end of the file (#343). */
export const START_KEY = 'start';
export const END_KEY = 'end';
@@ -47,6 +49,10 @@ export function loopsOf(bag: PropertyBag): boolean {
return booleanOf(bag[LOOP_KEY]);
}
export function streamsOf(bag: PropertyBag): boolean {
return booleanOf(bag[STREAM_KEY]);
}
/** The stretch of its file a clip is, its fades in seconds. */
export function cutOf(bag: PropertyBag): SoundCut {
const seconds = (key: string, fallback: number) => Math.max(0, coerceAs(bag[key], 'number', fallback));
src/core/store/serialize.ts
+13 -1
View File
@@ -22,6 +22,7 @@ import { isShaderNodeDef } from '../shader/nodes';
import { isSoundFxNodeDef } from '../soundfx/nodes';
import { INSTRUMENT_SAMPLE, isInstrumentNodeDef, withSamplePlay } from '../instrument/nodes';
import { heldBake } from '../music/bake';
import { heldStream } from '../music/streams';
import { heldPatternNotes } from '../music/pattern';
import { isEmbeddable } from '../music/embedded';
import { heldSongItems, heldSongTracks } from '../music/song';
@@ -263,7 +264,16 @@ export function deserializeProject(raw: unknown, fallbackId = ''): Project {
} else if (type === 'sound') {
const file = typeof a.file === 'string' && isSoundFile(a.file) ? a.file : '';
const clips = normalizeClips(a.clips, takenIds);
assets.push({ id, type, name, props, file, ...(clips.length > 0 && { clips }) });
const stream = heldStream(a.stream);
assets.push({
id,
type,
name,
props,
file,
...(clips.length > 0 && { clips }),
...(stream && { stream }),
});
} else if (type === 'tile') {
// A picture naming nothing is kept, like an animation's dangling root: it reads as missing
// in the tab and comes back when the texture it named is restored by an undo.
@@ -763,10 +773,12 @@ function normalizeClips(raw: unknown, takenIds: Set<string>): SoundClip[] {
const id = typeof c.id === 'string' ? c.id : '';
if (!id || takenIds.has(id)) continue;
takenIds.add(id);
const stream = heldStream(c.stream);
clips.push({
id,
name: typeof c.name === 'string' ? c.name : '',
props: normalizeBag(c.props, CLIP_ROWS, true),
...(stream && { stream }),
});
}
return clips;
src/editor/Editor.ts
+5 -1
View File
@@ -9,6 +9,7 @@ import { pluginsOf } from '../core/store/projectPlugins';
import type { FontStore } from '../core/store/FontStore';
import type { SnapshotStore } from '../core/store/SnapshotStore';
import type { SoundStore } from '../core/store/SoundStore';
import { StreamKeeper } from './sound/StreamKeeper';
import type { TextureStore } from '../core/store/TextureStore';
import type { UsersStore } from '../core/store/UsersStore';
import { gameClock, timeStep } from '../core/time';
@@ -249,6 +250,8 @@ export class Editor implements PlayHost {
readonly shaderEditor: ShaderEditor;
readonly particlesEditor: ParticlesEditor;
readonly soundEditor: SoundEditor;
/** Makes the copies streamed sounds play from (#349). */
readonly streamKeeper: StreamKeeper;
readonly soundFxEditor: SoundFxEditor;
readonly instrumentEditor: InstrumentEditor;
readonly patternEditor: PatternEditor;
@@ -349,7 +352,8 @@ export class Editor implements PlayHost {
this.stateMachineEditor = new StateMachineEditor(state);
this.shaderEditor = new ShaderEditor(state);
this.particlesEditor = new ParticlesEditor(state);
this.soundEditor = new SoundEditor(state);
this.streamKeeper = new StreamKeeper(state, sounds);
this.soundEditor = new SoundEditor(state, this.streamKeeper);
this.soundFxEditor = new SoundFxEditor(state);
this.instrumentEditor = new InstrumentEditor(state);
this.patternEditor = new PatternEditor(state, sounds);
src/editor/debug/soundHooks.ts
+59
View File
@@ -1,5 +1,7 @@
import { findAssetOf } from '../../core/model/Project';
import { clipsOf, findSound } from '../../core/model/sound';
import { STREAM_RATE } from '../../core/music/streamQueue';
import { playedPlan, streamTable } from '../../core/music/streams';
import { playedSoundOf } from '../../core/properties/playedSound';
import { soundLibrary } from '../../viewport/SoundLibrary';
import { centreOf, type HookContext } from './hookHelpers';
@@ -72,5 +74,62 @@ export function soundHooks({ state, editor }: HookContext) {
};
},
},
/** Streamed sounds (#349). */
streams: {
/** Done once every copy wanted has been made. */
idle: () => editor.streamKeeper.idle(),
/** Every name a game streams, with its copy. */
table: () => [...streamTable(state.project)].map(([name, copy]) => ({ name, ...copy })),
/**
* Streams `seconds` of a name offline from `offset` seconds in, heard from `at`, looped or not,
* and holds it against its copy decoded whole: the largest difference of any sample from what
* should be heard there – silence before `at` – against the feed's own decoder and against the
* browser's `decodeAudioData`, and how many samples came too late.
*/
exact: async (name: string, offset: number, loop: boolean, at: number, seconds: number) => {
const entry = streamTable(state.project).get(name);
if (!entry) return null;
const plan = playedPlan(entry.frames, offset, loop);
const frames = Math.round((at + seconds) * STREAM_RATE);
const [played, whole, browser] = await Promise.all([
soundLibrary.renderStream(name, plan, at, frames),
soundLibrary.streamedWhole(name),
soundLibrary.decodeApart(entry.file, STREAM_RATE),
]);
if (!played || !whole) return null;
const first = Math.round(at * STREAM_RATE);
const placeOf = (i: number) =>
loop ? (plan.from + i - first) % entry.frames : plan.from + i - first;
const worst = (reference: (c: number, place: number) => number) => {
let most = 0;
played.channels.forEach((side, c) => {
for (let i = 0; i < frames; i++) {
const place = placeOf(i);
const expected = i < first || place >= entry.frames ? 0 : reference(c, place);
most = Math.max(most, Math.abs(side[i]! - expected));
}
});
return most;
};
return {
frames: entry.frames,
decoded: browser.length,
own: worst((c, place) => whole[c]![place]!),
browser: worst(
(c, place) => browser.getChannelData(Math.min(c, browser.numberOfChannels - 1))[place]!,
),
skipped: played.skipped,
};
},
/** What a sound and each of its clips keep of their copies. */
copies: (id: string) => {
const asset = findAssetOf(state.project, id, 'sound');
if (!asset) return null;
return {
sound: asset.stream ?? null,
clips: clipsOf(asset).map((clip) => ({ id: clip.id, stream: clip.stream ?? null })),
};
},
},
};
}
src/editor/music/BakeButton.ts
+3 -15
View File
@@ -12,6 +12,7 @@ import { Button, formatBytes, Menu, Theme } from '../../ui';
import { BakeCancelled, makeBake } from '../../viewport/bakeRender';
import { HEADER_ICON_SVG } from '../panels/headerIcons';
import type { EditorState } from '../state';
import { dropWhenSaved } from './dropWhenSaved';
import { clockText } from './timeView';
/** Where a bake stands for the asset shown. */
@@ -128,7 +129,7 @@ export class BakeButton extends Button {
}
const old = this.current(asset.id)?.bake?.file;
this.state.setBake(asset.id, { file: uploaded.file, bytes: uploaded.bytes, ...rest });
if (old && old !== uploaded.file) this.dropWhenSaved(old);
if (old && old !== uploaded.file) dropWhenSaved(this.state, this.sounds, old);
this.state.setNotice(
`Baked ${asset.name}: ${clockText(rest.length / BAKE_RATE)}, ${formatBytes(uploaded.bytes)}`,
);
@@ -145,7 +146,7 @@ export class BakeButton extends Button {
private remove(asset: BakedAsset): void {
const old = asset.bake?.file;
this.state.setBake(asset.id, null);
if (old) this.dropWhenSaved(old);
if (old) dropWhenSaved(this.state, this.sounds, old);
}
private current(id: string): BakedAsset | null {
@@ -162,17 +163,4 @@ export class BakeButton extends Button {
private changed(asset: BakedAsset): void {
this.state.setNotice(`${asset.name} changed while it was baked – bake it again`, 'error');
}
/**
* Deletes the file of a bake that has gone, once the project no longer names it on the server: the
* server keeps a file any saved project or snapshot names (409), and so does this.
*/
private dropWhenSaved(file: string): void {
const off = this.state.on('save', () => {
const status = this.state.saveStatus;
if (status === 'dirty' || status === 'saving') return;
off();
if (status === 'saved') void this.sounds.delete(file).catch(() => {});
});
}
}
src/editor/music/dropWhenSaved.ts
+16
View File
@@ -0,0 +1,16 @@
import type { SoundStore } from '../../core/store/SoundStore';
import type { EditorState } from '../state';
/**
* Deletes a file of the sounds a bake (#340) or a stream copy (#349) left behind, once the project no
* longer names it on the server: the server keeps a file any saved project or snapshot names (409),
* and so does this.
*/
export function dropWhenSaved(state: EditorState, sounds: SoundStore, file: string): void {
const off = state.on('save', () => {
const status = state.saveStatus;
if (status === 'dirty' || status === 'saving') return;
off();
if (status === 'saved') void sounds.delete(file).catch(() => {});
});
}
src/editor/song/SongTimeline.ts
+2 -2
View File
@@ -1368,9 +1368,9 @@ export class SongTimeline extends Widget {
}
private ensureLoaded(file: string): void {
if (!file || this.loading.has(file) || soundLibrary.duration(file) !== null) return;
if (!file || this.loading.has(file) || soundLibrary.peaks(file) !== null) return;
this.loading.add(file);
void soundLibrary.load(file).then(() => this.invalidate());
void soundLibrary.loadPicture(file).then(() => this.invalidate());
}
override layout(): void {
src/editor/sound/SoundEditor.ts
+23 -2
View File
@@ -12,8 +12,10 @@ import {
FADE_OUT_CURVE_KEY,
FADE_OUT_KEY,
REVERSE_KEY,
streamsOf,
volumeOf,
} from '../../core/properties/sounds';
import { STREAM_SUGGESTED } from '../../core/music/streams';
import { Button, Dropdown, Label, Menu, NumberField } from '../../ui';
import { soundLibrary } from '../../viewport/SoundLibrary';
import { MASTER_MIX } from '../../core/music/mixer';
@@ -26,6 +28,7 @@ import { claimTransport, releaseTransport } from '../music/transport';
import { HEADER_ICON_SVG } from '../panels/headerIcons';
import { TOOLBAR_ICON_SVG } from '../panels/toolbarIcons';
import type { EditorState } from '../state';
import type { StreamKeeper } from './StreamKeeper';
import { WaveCanvas } from './WaveCanvas';
const INFO_WIDTH = 140;
@@ -71,7 +74,10 @@ export class SoundEditor extends AssetTabEditor<SoundAsset, WaveCanvas> {
private playingId = '';
private readonly halt = () => this.stop();
constructor(state: EditorState) {
constructor(
state: EditorState,
private readonly streams: StreamKeeper,
) {
super(state, { infoWidth: INFO_WIDTH });
this.playButton = new Button('Play', {
icon: TOOLBAR_ICON_SVG.play,
@@ -125,6 +131,8 @@ export class SoundEditor extends AssetTabEditor<SoundAsset, WaveCanvas> {
state.on('load', () => this.stop());
// Picking a clip changes what the header plays and edits.
state.on('selection', () => this.refresh());
// How far a stream copy has got (#349) comes as a notice, not as a change of the project.
state.on('notice', () => this.refresh());
}
override selectedId(): string {
@@ -251,7 +259,20 @@ export class SoundEditor extends AssetTabEditor<SoundAsset, WaveCanvas> {
if (!asset.file) return 'Not uploaded';
if (!span || whole === null) return '';
const kept = span.end - span.start;
return kept >= whole ? `${whole.toFixed(2)} s` : `${kept.toFixed(2)} s of ${whole.toFixed(2)} s`;
const length = kept >= whole ? `${whole.toFixed(2)} s` : `${kept.toFixed(2)} s of ${whole.toFixed(2)} s`;
return `${length}${this.streamText(asset, whole)}`;
}
/**
* Whether it streams (#349): its copy is made, being made or yet to be; a long one that does not
* is pointed at the switch.
*/
private streamText(asset: SoundAsset, whole: number): string {
const making = this.streams.making(asset.id);
if (making !== null) return ` · copy ${Math.round(making * 100)}%`;
if (streamsOf(resolveAssetProps(this.state.project, asset)))
return asset.stream?.from === asset.file ? ' · streamed' : ' · copy waiting';
return whole >= STREAM_SUGGESTED ? ' · long, try Stream' : '';
}
protected override syncExtras(asset: SoundAsset | null): void {
src/editor/sound/SoundPicker.ts
+4 -1
View File
@@ -1,5 +1,6 @@
import { Container, type FederatedPointerEvent, Graphics, type Text } from 'pixi.js';
import { isBakeUpload } from '../../core/music/bake';
import { isStreamUpload } from '../../core/music/streams';
import type { StoredSound } from '../../core/store/SoundStore';
import { drawSvg, formatBytes, textStyle, Theme, truncate, crispText } from '../../ui';
import { soundLibrary } from '../../viewport/SoundLibrary';
@@ -121,7 +122,9 @@ class SoundPicker extends FilePicker<SoundItem> {
/** A bake (#340) is in the sounds, but no sound to pick. */
protected async fetchItems(): Promise<SoundItem[]> {
const sounds = (await this.deps.sounds.list()).filter((sound) => !isBakeUpload(sound.name));
const sounds = (await this.deps.sounds.list()).filter(
(sound) => !isBakeUpload(sound.name) && !isStreamUpload(sound.name),
);
return withAssetNames(sounds, this.deps.state.project, 'sound');
}
src/editor/sound/StreamKeeper.ts
+164
View File
@@ -0,0 +1,164 @@
import type { SoundAsset } from '../../core/model/Asset';
import { findAsset } from '../../core/model/Project';
import { clipsOf, cutName, cutChannels, cutRange, parseCutName, soundRefName } from '../../core/model/sound';
import { STREAM_RATE } from '../../core/music/streamQueue';
import { type StreamJob, streamUploadName, streamWork } from '../../core/music/streams';
import { cutOf, streamsOf } from '../../core/properties/sounds';
import type { SoundStore } from '../../core/store/SoundStore';
import { BakeCancelled, encodeOpus } from '../../viewport/bakeRender';
import { soundLibrary } from '../../viewport/SoundLibrary';
import { dropWhenSaved } from '../music/dropWhenSaved';
import type { EditorState } from '../state';
/** Milliseconds the project has to stay as it is before a copy is made: a clip dragged about is one. */
const SETTLE_MS = 600;
/**
* Makes the copies streamed sounds are played from (#349). A moment after the project last changed,
* each sound with `stream` on gets a copy of its file as Opus, and each of its clips one of its cut,
* one at a time, from the file decoded for it and let go again. A copy no longer wanted is taken off
* its sound and deleted once the project has been saved without it. Not while the game runs.
*/
export class StreamKeeper {
private job: { job: StreamJob; share: number } | null = null;
private timer: ReturnType<typeof setTimeout> | null = null;
/** Copies that could not be made, so one broken file is not tried over and over. */
private readonly failed = new Set<string>();
private waiting: (() => void)[] = [];
constructor(
private readonly state: EditorState,
private readonly sounds: SoundStore,
) {
for (const event of ['project', 'load', 'mode'] as const) state.on(event, () => this.later());
this.later();
}
/** How far the copy of that sound, or of one of its clips, has got, 0 to 1; null while none is made. */
making(assetId: string): number | null {
return this.job?.job.assetId === assetId ? this.job.share : null;
}
/** Done once there is nothing left to make (test hook). */
idle(): Promise<void> {
if (!this.job && this.timer === null && this.pending().length === 0) return Promise.resolve();
return new Promise((resolve) => this.waiting.push(resolve));
}
private later(wait = SETTLE_MS): void {
if (this.timer !== null) clearTimeout(this.timer);
this.timer = setTimeout(() => {
this.timer = null;
void this.work();
}, wait);
}
private pending(): StreamJob[] {
return streamWork(this.state.project).jobs.filter((job) => !this.failed.has(keyOf(job)));
}
private async work(): Promise<void> {
if (this.job || this.state.mode === 'play') return;
for (const drop of streamWork(this.state.project).drops) {
const old = this.ownerOf(drop.assetId, drop.clipId)?.stream?.file;
this.state.setStream(drop.assetId, drop.clipId, null);
if (old && !this.named(old)) dropWhenSaved(this.state, this.sounds, old);
}
const next = this.pending()[0];
if (!next) {
for (const resolve of this.waiting.splice(0)) resolve();
return;
}
this.job = { job: next, share: 0 };
try {
await this.make(this.job);
} catch (error: unknown) {
// Called off because what it was made of changed: the next round makes the right one.
if (error instanceof BakeCancelled) return this.later(0);
this.failed.add(keyOf(next));
this.state.setNotice(
`No stream copy of ${this.label(next)}: ${error instanceof Error ? error.message : String(error)}`,
'error',
);
} finally {
this.job = null;
}
this.later(0);
}
private async make(running: { job: StreamJob; share: number }): Promise<void> {
const { job } = running;
const label = this.label(job);
const { file, cut } = parseCutName(job.from);
const whole = await soundLibrary.decodeApart(file, STREAM_RATE);
let channels = Array.from({ length: Math.min(2, whole.numberOfChannels) }, (_, c) =>
whole.getChannelData(c),
);
if (cut) channels = cutChannels(channels, cutRange(whole.length, whole.sampleRate, cut));
const data = await encodeOpus(channels, {
progress: (share) => {
const before = Math.round(running.share * 100);
running.share = share;
const now = Math.round(share * 100);
if (now !== before) this.state.setNotice(`Making a stream copy of ${label}: ${now}%`, 'quiet');
},
cancelled: () => !this.stillWanted(job),
});
if (!this.stillWanted(job)) return;
const uploaded = await this.sounds.upload(data.slice().buffer, 'audio/ogg', streamUploadName(label));
// Nothing names the file yet, so it can go at once.
if (!this.stillWanted(job)) return void this.sounds.delete(uploaded.file).catch(() => {});
const old = this.ownerOf(job.assetId, job.clipId)?.stream?.file;
const frames = channels[0]?.length ?? 0;
// Done: what the Sound tab reads when the copy lands is that it streams, not how far it got.
this.job = null;
this.state.setStream(job.assetId, job.clipId, {
file: uploaded.file,
from: job.from,
frames,
bytes: uploaded.bytes,
});
if (old && old !== uploaded.file && !this.named(old)) dropWhenSaved(this.state, this.sounds, old);
this.state.setNotice(`Made a stream copy of ${label}`, 'quiet');
}
/** Whether the sound still streams and still plays what the copy is being made of. */
private stillWanted(job: StreamJob): boolean {
const asset = this.sound(job.assetId);
if (!asset || !streamsOf(asset.props)) return false;
if (job.clipId === null) return asset.file === job.from;
const clip = clipsOf(asset).find((c) => c.id === job.clipId);
return clip !== undefined && cutName(asset.file, cutOf(clip.props)) === job.from;
}
/** Whether a copy is still named by another sound or clip – a sound and its duplicate share one. */
private named(file: string): boolean {
return this.state.project.assets.some(
(asset) =>
asset.type === 'sound' &&
(asset.stream?.file === file || clipsOf(asset).some((clip) => clip.stream?.file === file)),
);
}
private sound(id: string): SoundAsset | null {
const asset = findAsset(this.state.project, id);
return asset?.type === 'sound' ? asset : null;
}
private ownerOf(assetId: string, clipId: string | null): { stream?: SoundAsset['stream'] } | null {
const asset = this.sound(assetId);
if (!asset) return null;
return clipId === null ? asset : (clipsOf(asset).find((clip) => clip.id === clipId) ?? null);
}
private label(job: StreamJob): string {
const asset = this.sound(job.assetId);
if (!asset) return 'a sound';
const clip = clipsOf(asset).find((c) => c.id === job.clipId) ?? null;
return soundRefName({ asset, clip });
}
}
function keyOf(job: StreamJob): string {
return `${job.assetId}:${job.clipId ?? ''}:${job.from}`;
}
src/editor/state.ts
+18 -1
View File
@@ -6,7 +6,7 @@ import type { ObjectPick, PickEnd } from './objectPick';
import { findKey, findTrack, hasKey, type KeyRef, sameKey } from '../core/model/animation';
import { type Command, CommandError } from '../core/commands/Command';
import { CommandStack } from '../core/commands/CommandStack';
import type { Asset, GraphAsset, SceneAsset } from '../core/model/Asset';
import type { Asset, GraphAsset, SceneAsset, SoundStream } from '../core/model/Asset';
import { findAnnotation, type Graph } from '../core/graph';
import { findObject } from '../core/model/hierarchy';
import { allShapes } from '../core/model/vectorTree';
@@ -443,6 +443,23 @@ export class EditorState {
this.emit('project');
}
/**
* The copy a sound or one of its clips streams from (#349), or none: saved, and like a bake no undo
* step – it is made again whenever it no longer fits.
*/
setStream(assetId: string, clipId: string | null, stream: SoundStream | null): void {
for (const project of [this.project, this.edit?.project]) {
const asset = project && findAsset(project, assetId);
if (asset?.type !== 'sound') continue;
const owner = clipId === null ? asset : clipsOf(asset).find((clip) => clip.id === clipId);
if (!owner) continue;
if (stream) owner.stream = stream;
else delete owner.stream;
}
this.setSaveStatus('dirty');
this.emit('project');
}
/**
* Runs a command through the undo stack. Returns false (and sets a notice) when invalid.
* In play mode the command changes the runtime copy only (see `enterPlay`).
src/viewport/SoundLibrary.ts
+386 -88
View File
@@ -31,11 +31,13 @@ import { limiterLatency } from '../core/soundfx/dynamics';
import type { ScheduledClip, ScheduledNote } from '../core/music/Scheduler';
import { dbToGain, SOUND_ITEM_MAX_DB } from '../core/music/song';
import { DEFAULT_TEMPO } from '../core/music/time';
import { playedPlan, type StreamedFile, type StreamPlan } from '../core/music/streams';
import { STREAM_RATE } from '../core/music/streamQueue';
import { type CutRange, cutChannels, cutRange, parseCutName } from '../core/model/sound';
import { shifterLatency } from '../core/soundfx/pitchPlayer';
import { REVERB_MAX_DECAY } from '../core/soundfx/reverb';
import type { AudioSink, ClipTiming, NoteTiming } from '../runtime/systems/Audio';
import type { ChipOptions, PitchOptions } from './fxWorklet';
import type { ChipOptions, PitchOptions, StreamOptions } from './fxWorklet';
import fxWorkletUrl from './fxWorklet.ts?worker&url';
import {
bothAtCenter,
@@ -57,6 +59,7 @@ import {
stopProcessor,
} from './fxNodes';
import { type Spectrum, SpectrumTaps } from './spectrum';
import { decodeSpan, StreamFeed, StreamFile } from './streaming';
import { wake } from './wake';
/** A voice is no louder than a sound item of a song turned all the way up. */
@@ -65,6 +68,8 @@ const MAX_VOLUME = dbToGain(SOUND_ITEM_MAX_DB);
const DEFAULT_BASE = '/game-maker/data/sounds';
/** The rate a sound loaded ahead of its first play is decoded at; a source resamples it on the way out. */
const DECODE_RATE = 48000;
/** The rate the picture of a streamed sound is decoded at (#349): enough for its peaks, a sixth of the size. */
const PICTURE_RATE = 8000;
/** The longest a send of a track is held back to meet the latest way of it (#350). */
const MAX_SEND_DELAY = 1;
@@ -199,6 +204,8 @@ interface Wiring {
/** Seconds into the sound its ways start, and whether it is a clip of a song, which fades out at its end (#334). */
offset: number;
clip: boolean;
/** The copy a way without a source of its own streams (#349), instead of a buffer of `file`. */
stream: StreamFile | null;
sources: Source[];
drivers: Driver[];
envelopes: Map<string, EnvelopeRun>;
@@ -229,6 +236,8 @@ interface Start {
clip?: { offset: number; end: number };
/** The track of a song it is heard on (#334), `''` straight out. */
bus?: string;
/** The copy `file` streams from (#349); its bytes are not loaded then, its head is. */
stream?: StreamFile;
}
/**
@@ -262,6 +271,9 @@ interface Pending {
*
* A way with a Pitch is played by a worklet (#317), loaded into the context with the first play that
* needs it; that play waits for it as it would for its bytes.
*
* A sound that streams (#349) is the exception to "decoded whole": a `stream-player` on that worklet
* plays it from its copy, fed a few seconds ahead (`streaming.ts`), and its file is never decoded.
*/
export class SoundLibrary implements AudioSink {
private base = DEFAULT_BASE;
@@ -331,6 +343,7 @@ export class SoundLibrary implements AudioSink {
/** The taps of the voices and buses playing now, and the node listened to (#374). */
private readonly spectra = new SpectrumTaps();
private readonly peaked = new Map<string, Float32Array>();
private readonly pictures = new Map<string, Promise<void>>();
/**
* The patterns and songs a game plays, each on a scheduler of its own. What it asks after are its
* own voices – asking `playing`, which counts the playings too, would ask the list it is in.
@@ -363,12 +376,40 @@ export class SoundLibrary implements AudioSink {
private mix: readonly MixBus[] = [];
/** The levels a running game set its buses to, over those of their assets; until `resetMix`. */
private readonly mixVolumes = new Map<string, number>();
/** The names streamed (#349) and the copies they stream from, by name and by copy. */
private streams: ReadonlyMap<string, StreamedFile> = new Map();
private readonly streamFiles = new Map<string, StreamFile>();
private readonly feeds = new Set<StreamFeed>();
/** Every sample streamed so far that came too late to be heard, of the feeds gone (test hook). */
private skippedBefore = 0;
/** Where the files are served from. Set once at start-up, before anything plays. */
configure(base: string): void {
this.base = base.replace(/\/+$/, '');
}
/**
* The names a game streams rather than decodes (#349), each with its copy (`streamTable`): the first
* seconds of every copy are fetched and decoded now, so it starts at once. A library that bakes,
* and a browser without Opus in WebCodecs, decode everything as before.
*/
setStreams(table: ReadonlyMap<string, StreamedFile>): void {
this.streams = table;
for (const name of table.keys()) void this.streamed(name)?.ready();
}
/** The copy a name streams from, while it streams and the copy has not failed. */
private streamed(name: string): StreamFile | null {
const entry = this.streams.get(name);
if (!entry || this.render || typeof AudioDecoder === 'undefined') return null;
let file = this.streamFiles.get(entry.file);
if (!file) {
file = new StreamFile(`${this.base}/${entry.file}`, entry.frames);
this.streamFiles.set(entry.file, file);
}
return file.failed ? null : file;
}
/**
* The number is handed out at once, before the bytes are there: a graph can tune a voice in the
* same frame it asked for it, and the voice then starts with those values.
@@ -376,6 +417,7 @@ export class SoundLibrary implements AudioSink {
play(file: string, volume: number, loop: boolean, owner = '', fx: FxVoice = DRY_VOICE, bus = ''): number {
if (file === '' || this.hasFailed(file)) return 0;
const layered = { layers: [fx.plan], params: fx.params, tempo: fx.tempo ?? DEFAULT_TEMPO };
const stream = this.streamed(file);
return this.begin({
id: 0,
file,
@@ -387,6 +429,7 @@ export class SoundLibrary implements AudioSink {
duration: 0,
at: 0,
bus,
...(stream && { stream }),
});
}
@@ -424,6 +467,7 @@ export class SoundLibrary implements AudioSink {
if (file === '' || this.hasFailed(file)) return 0;
const layered = { layers: [fx.plan], params: fx.params, tempo: fx.tempo ?? DEFAULT_TEMPO };
const { at, offset, duration, volume, bus } = clip;
const stream = this.streamed(file);
return this.begin({
id: 0,
file,
@@ -436,6 +480,7 @@ export class SoundLibrary implements AudioSink {
at,
clip: { offset, end: at + Math.max(0, duration) },
bus,
...(stream && { stream }),
});
}
@@ -1050,19 +1095,102 @@ export class SoundLibrary implements AudioSink {
/**
* The bake of a playing (#340) as one voice on its master, heard from `heard`: its file once to its
* end, faded out there, or up to the loop end and then round its second round – in one source where
* the round follows at once, else in a second that takes over to the sample. 0 without its bytes.
* end, faded out there, or up to the loop end and then round its second round. Streamed where it can
* be (#349), else decoded – in one source where the round follows at once, else in a second that
* takes over to the sample. 0 without its bytes.
*/
private playBaked(id: number, bake: BakePlay, heard: number): number {
const buffer = this.buffer(bake.file);
if (!buffer) return 0;
const stream = this.hasWorklet ? this.streamed(bake.file) : null;
const streaming = stream?.head ? stream : null;
const buffer = streaming ? null : this.buffer(bake.file);
if (!streaming && !buffer) return 0;
const context = this.audio();
const bus = groupBus(id, '');
const at = Math.max(context.currentTime, heard - this.busLatency(bus));
const gain = context.createGain();
gain.gain.value = Math.max(0, Math.min(MAX_VOLUME, bake.gain));
gain.connect(this.busInput(bus));
const { end, until, round } = bake.playback;
const { end } = bake.playback;
const played = streaming
? this.bakeStreamed(streaming, bake.playback, gain, at)
: this.bakeDecoded(buffer!, bake.playback, gain, at);
const voice: Voice = {
id: this.nextId++,
file: bake.file,
owner: '',
bus,
at,
sources: played.sources,
ending: new Set(),
drivers: [],
envelopes: [],
finish: null,
gain,
fx: { layers: [], params: {}, tempo: DEFAULT_TEMPO },
knobs: [],
rings: false,
ringing: false,
quiet: null,
vocoders: 0,
released: false,
ringFade: BAKE_LET_RING,
};
const going = new Set(voice.sources);
voice.finish = (source) => {
if (going.delete(source) && going.size === 0) this.ended(voice);
};
for (const source of played.sources) source.ended(() => voice.finish?.(source));
this.voices.push(voice);
this.started += 1;
this.last = {
file: bake.file,
volume: gain.gain.value,
loop: bake.playback.round !== null,
layers: 0,
pitches: [],
stages: 0,
duration: played.duration,
at,
envelopes: 0,
bus,
};
if (end !== null) {
gain.gain.setValueAtTime(gain.gain.value, at + Math.max(0, end - FADE));
gain.gain.linearRampToValueAtTime(0, at + end);
}
played.start();
return voice.id;
}
/** A bake streamed (#349): one `stream-player` along the same plan, the round looped to the sample. */
private bakeStreamed(
stream: StreamFile,
{ end, until, round }: BakePlay['playback'],
gain: GainNode,
at: number,
): { sources: Source[]; start: () => void; duration: number } {
const frames = (seconds: number) => Math.round(seconds * STREAM_RATE);
const plan: StreamPlan =
end !== null
? { from: 0, until: frames(end), loop: null }
: {
from: 0,
until: frames(until),
loop: round && { start: frames(round.start), end: frames(round.end) },
};
const { node, source } = this.streamNode(this.audio(), stream, plan, at);
node.connect(gain);
return { sources: [source], start: () => {}, duration: stream.frames / STREAM_RATE };
}
/** A bake decoded, in one buffer source or two. */
private bakeDecoded(
buffer: AudioBuffer,
{ end, until, round }: BakePlay['playback'],
gain: GainNode,
at: number,
): { sources: Source[]; start: () => void; duration: number } {
const context = this.audio();
const made = (from: number, loop: { start: number; end: number } | null) => {
const node = context.createBufferSource();
node.buffer = buffer;
@@ -1090,56 +1218,15 @@ export class SoundLibrary implements AudioSink {
: round
? [made(0, null), made(round.start, round)]
: [made(0, null)];
const voice: Voice = {
id: this.nextId++,
file: bake.file,
owner: '',
bus,
at,
sources: parts.map((part) => part.source),
ending: new Set(),
drivers: [],
envelopes: [],
finish: null,
gain,
fx: { layers: [], params: {}, tempo: DEFAULT_TEMPO },
knobs: [],
rings: false,
ringing: false,
quiet: null,
vocoders: 0,
released: false,
ringFade: BAKE_LET_RING,
};
const going = new Set(voice.sources);
voice.finish = (source) => {
if (going.delete(source) && going.size === 0) this.ended(voice);
};
for (const { source } of parts) source.ended(() => voice.finish?.(source));
this.voices.push(voice);
this.started += 1;
this.last = {
file: bake.file,
volume: gain.gain.value,
loop: round !== null,
layers: 0,
pitches: [],
stages: 0,
duration: buffer.duration,
at,
envelopes: 0,
bus,
};
const [head, next] = parts;
if (end !== null) {
gain.gain.setValueAtTime(gain.gain.value, at + Math.max(0, end - FADE));
gain.gain.linearRampToValueAtTime(0, at + end);
head!.node.start(at, 0, end);
} else if (next) {
head!.node.start(at, 0, until);
next.node.start(at + until, next.from);
} else head!.node.start(at, 0);
return voice.id;
const start = () => {
if (end !== null) head!.node.start(at, 0, end);
else if (next) {
head!.node.start(at, 0, until);
next.node.start(at + until, next.from);
} else head!.node.start(at, 0);
};
return { sources: parts.map((part) => part.source), start, duration: buffer.duration };
}
/**
@@ -1304,8 +1391,12 @@ export class SoundLibrary implements AudioSink {
const asked: Start = { ...start, id };
// A Reverb's sound (#328) is waited for as well, but a voice is not silent for lack of it.
const impulseFiles = start.fx.layers.flatMap(impulsesOf);
const loaded = [...start.files, ...impulseFiles].every((file) => this.buffer(file) !== null);
const worklet = start.fx.layers.some(needsWorklet) ? this.fxWorklet() : null;
// A streamed sound (#349) waits for the head of its copy and the worklet that plays it instead.
const { stream } = start;
const loaded =
[...(stream ? [] : start.files), ...impulseFiles].every((file) => this.buffer(file) !== null) &&
(!stream || stream.head !== null);
const worklet = stream || start.fx.layers.some(needsWorklet) ? this.fxWorklet() : null;
if (loaded && (!worklet || this.hasWorklet !== null)) {
this.start(asked);
return id;
@@ -1321,19 +1412,23 @@ export class SoundLibrary implements AudioSink {
};
this.pending.push(waiting);
const impulsesLoaded = Promise.all(impulseFiles.map((file) => this.load(file)));
void Promise.all([Promise.all(start.files.map((file) => this.load(file))), worklet, impulsesLoaded]).then(
([buffers]) => {
this.pending = this.pending.filter((other) => other !== waiting);
// A `Stop sound` – or the destruction of the behavior that asked – that happened while the
// bytes were in flight has to win, or a sound the run already ended would start afterwards.
const silent = buffers.every((buffer) => !buffer) && !start.fx.layers.some(hasChip);
if (silent || waiting.stopped) return;
if (this.cancelsFor(start.file, start.owner) !== askedAt) return;
this.start({ ...asked, fx: waiting.fx });
for (const { params, time, ease } of waiting.timed) this.tune(id, params, time, ease);
if (waiting.released) this.release(id);
},
);
// A copy that cannot be streamed leaves the sound decoded, as it would be without one.
const sounds = stream
? stream.ready().then(async (ready) => [ready || (await this.load(start.file)) !== null])
: Promise.all(start.files.map((file) => this.load(file)));
void Promise.all([sounds, worklet, impulsesLoaded]).then(([buffers]) => {
this.pending = this.pending.filter((other) => other !== waiting);
// A `Stop sound` – or the destruction of the behavior that asked – that happened while the
// bytes were in flight has to win, or a sound the run already ended would start afterwards.
const silent = buffers.every((buffer) => !buffer) && !start.fx.layers.some(hasChip);
if (silent || waiting.stopped) return;
if (this.cancelsFor(start.file, start.owner) !== askedAt) return;
const started: Start = { ...asked, fx: waiting.fx };
if (stream?.failed) delete started.stream;
this.start(started);
for (const { params, time, ease } of waiting.timed) this.tune(id, params, time, ease);
if (waiting.released) this.release(id);
});
return id;
}
@@ -1463,7 +1558,13 @@ export class SoundLibrary implements AudioSink {
/** Ways playing through the pitch worklet now, at most `MAX_VOCODERS`. */
vocoders: number;
last: LastVoice | null;
/**
* Streaming (#349): sources playing from a copy now, samples that came too late to be heard, the
* bytes fetched of every copy, and the copies whose head is decoded.
*/
streaming: { playing: number; skipped: number; fetched: number; heads: number };
} {
const files = [...this.streamFiles.values()];
return {
playing: this.voices.length,
started: this.started,
@@ -1475,6 +1576,12 @@ export class SoundLibrary implements AudioSink {
worklet: this.hasWorklet,
vocoders: this.voices.reduce((sum, voice) => sum + vocodersOf(voice), 0),
last: this.last,
streaming: {
playing: this.feeds.size,
skipped: [...this.feeds].reduce((sum, feed) => sum + feed.skipped, this.skippedBefore),
fetched: files.reduce((sum, file) => sum + file.bytes, 0),
heads: files.filter((file) => file.head !== null).length,
},
};
}
@@ -1538,11 +1645,21 @@ export class SoundLibrary implements AudioSink {
* not late waiting on its bytes (#332).
*/
async prepare(layers: readonly FxPlan[], files: readonly string[] = []): Promise<void> {
// A Sample or a Reverb's sound is always decoded; a sound or a bake may stream (#349).
const decoded = new Set([...layers.flatMap(filesOf), ...layers.flatMap(impulsesOf)]);
const streamed = files.flatMap((file) => {
const stream = decoded.has(file) ? null : this.streamed(file);
return stream ? [{ file, stream }] : [];
});
const loaded = [...new Set([...decoded, ...files])].filter(
(file) => file !== '' && !streamed.some((entry) => entry.file === file),
);
await Promise.all([
...[...new Set([...layers.flatMap(filesOf), ...layers.flatMap(impulsesOf), ...files])]
.filter((file) => file !== '')
.map((file) => this.load(file)),
layers.some(needsWorklet) ? this.fxWorklet() : null,
...loaded.map((file) => this.load(file)),
...streamed.map(async ({ file, stream }) => {
if (!(await stream.ready())) await this.load(file);
}),
layers.some(needsWorklet) || streamed.length > 0 ? this.fxWorklet() : null,
]);
}
@@ -1562,7 +1679,10 @@ export class SoundLibrary implements AudioSink {
*/
duration(file: string): number | null {
const range = this.rangeOf(file);
return range ? (range.to - range.from) / range.whole.sampleRate : null;
if (range) return (range.to - range.from) / range.whole.sampleRate;
// A streamed one (#349) is as long as its copy, known without decoding a sample of it.
const stream = this.streams.get(file);
return stream ? stream.frames / STREAM_RATE : null;
}
/** The decoded samples of a file as it is, without a cut; null while it has not been loaded. */
@@ -1588,27 +1708,49 @@ export class SoundLibrary implements AudioSink {
if (known) return known;
const range = this.rangeOf(file);
if (!range) return null;
const { whole, from, to, reverse } = range;
const peaks = new Float32Array(PEAK_BUCKETS);
for (let c = 0; c < whole.numberOfChannels; c++) {
const data = whole.getChannelData(c);
for (let i = from; i < to; i++) {
const heard = reverse ? to - 1 - i : i - from;
const bucket = Math.floor((heard / (to - from)) * PEAK_BUCKETS);
peaks[bucket] = Math.max(peaks[bucket]!, Math.abs(data[i]!));
}
}
for (let b = 0; b < PEAK_BUCKETS; b++) peaks[b] = Math.min(1, peaks[b]!);
const peaks = peaksOf(range.whole, range);
this.peaked.set(file, peaks);
return peaks;
}
/**
* Loads what `peaks` and `duration` read for a name. A streamed one (#349) is not loaded whole for
* it: its copy is decoded at `PICTURE_RATE`, its peaks kept and the samples let go.
*/
loadPicture(name: string): Promise<void> {
const stream = this.streamed(name);
if (!stream) return this.load(name).then(() => {});
if (this.peaked.has(name)) return Promise.resolve();
let picture = this.pictures.get(name);
if (!picture) {
picture = this.draw(name, stream);
this.pictures.set(name, picture);
}
return picture;
}
private async draw(name: string, stream: StreamFile): Promise<void> {
try {
const bytes = await stream.whole();
const picture = await new OfflineAudioContext(1, 1, PICTURE_RATE).decodeAudioData(bytes);
this.peaked.set(name, peaksOf(picture, { from: 0, to: picture.length, reverse: false }));
} catch (error: unknown) {
console.error(`[sounds] cannot draw ${name}`, error);
}
}
/**
* Starts loading every file not held yet. Waiting for the first play left that play late: its
* start was not heard, and a graph tuning it in the meantime had moved on by the time it started.
* One that streams (#349) has the head of its copy loaded instead.
*/
preload(files: Iterable<string>): void {
for (const file of files) if (file) void this.load(file);
for (const file of files) {
if (!file) continue;
const stream = this.streamed(file);
if (stream) void stream.ready();
else void this.load(file);
}
}
/**
@@ -1643,6 +1785,13 @@ export class SoundLibrary implements AudioSink {
return work;
}
/** A file decoded at `rate` and not kept: what a stream copy is made from (#349). */
async decodeApart(file: string, rate: number): Promise<AudioBuffer> {
const response = await fetch(`${this.base}/${file}`);
if (!response.ok) throw new Error(`HTTP ${response.status}`);
return new OfflineAudioContext(1, 1, rate).decodeAudioData(await response.arrayBuffer());
}
private async fetched(file: string): Promise<AudioBuffer> {
const response = await fetch(`${this.base}/${file}`);
if (!response.ok) throw new Error(`HTTP ${response.status}`);
@@ -1806,6 +1955,7 @@ export class SoundLibrary implements AudioSink {
at: heard,
clip: asked,
bus = '',
stream,
}: Start): void {
const context = this.audio();
// A context starts suspended where the page has not been interacted with; without this the
@@ -1840,6 +1990,7 @@ export class SoundLibrary implements AudioSink {
rings,
offset: clip ? clip.offset + (when - at) : 0,
clip: clip !== undefined,
stream: stream ?? null,
sources: [],
drivers: [],
envelopes: new Map(),
@@ -1990,7 +2141,15 @@ export class SoundLibrary implements AudioSink {
return sound.connect(fade);
}
private sound(wiring: Wiring, { pitch, shifted, source: own, pitchMods }: Way): AudioNode {
private sound(wiring: Wiring, way: Way): AudioNode {
const { pitch, shifted, source: own, pitchMods } = way;
if (!own && wiring.stream && this.hasWorklet)
return this.streamSource(
wiring,
wiring.stream,
playedPlan(wiring.stream.frames, wiring.offset, wiring.loop),
way,
);
const { context } = wiring;
const file = own?.file ?? wiring.file;
const loop = own ? own.loop : wiring.loop;
@@ -2063,6 +2222,127 @@ export class SoundLibrary implements AudioSink {
return source;
}
/**
* A way streamed from its copy (#349): a `stream-player` fed along its plan. Its Pitch is a shifter
* of what it plays, as on a bus (#346): its length stays, and it is heard a frame late.
*/
private streamSource(
wiring: Wiring,
file: StreamFile,
plan: StreamPlan,
{ pitch, shifted, pitchMods }: Way,
): AudioNode {
const { context } = wiring;
wiring.pitches.push(pitch);
const { node, source } = this.streamNode(context, file, plan, wiring.at);
wiring.sources.push(source);
if (!shifted) return node;
const shifter = new AudioWorkletNode(context, 'pitch-shifter', {
numberOfInputs: 1,
numberOfOutputs: 1,
outputChannelCount: [2],
channelCount: 2,
channelCountMode: 'explicit',
parameterData: { pitch },
});
node.connect(shifter);
wiring.vocoders += 1;
wiring.drivers.push({ start: () => {}, stop: () => stopProcessor(shifter) });
const knob = shifter.parameters.get('pitch')!;
wiring.knobs.push({ param: knob, target: pitch, even: 'value' });
const follow = this.follower(wiring);
for (const mod of pitchMods ?? []) follow(mod, knob, false);
return shifter;
}
/** A `stream-player` fed along `plan`, heard from `at` on, and the source a voice holds it by. */
private streamNode(
context: BaseAudioContext,
file: StreamFile,
plan: StreamPlan,
at: number,
): { node: AudioWorkletNode; source: Source; feed: StreamFeed } {
let then: (() => void) | null = null;
const feed: StreamFeed = new StreamFeed(file, plan, () => {
this.dropFeed(feed);
then?.();
});
const options: StreamOptions = { channels: file.channels, startAt: at, first: feed.opening() };
const node = new AudioWorkletNode(context, 'stream-player', {
numberOfInputs: 0,
outputChannelCount: [file.channels],
processorOptions: options,
});
feed.attach(node);
this.feeds.add(feed);
const source: Source = {
start: () => {},
stop: () => {
this.dropFeed(feed);
feed.stop();
},
ended: (call) => {
then = call;
},
fade: null,
gate: false,
leave: null,
};
return { node, source, feed };
}
/**
* Streams a name along `plan` into an offline context at `STREAM_RATE`, heard from `at`, `frames`
* long, with the player and the feed a game has (test hook). The clock is held every half second
* until the feed has caught up: offline it runs faster than anything is decoded. Null where the
* name does not stream.
*/
async renderStream(
name: string,
plan: StreamPlan,
at: number,
frames: number,
): Promise<{ channels: Float32Array[]; skipped: number } | null> {
const file = this.streamed(name);
if (!file || !(await file.ready())) return null;
const context = new OfflineAudioContext(file.channels, frames, STREAM_RATE);
await context.audioWorklet.addModule(fxWorkletUrl);
const { node, source, feed } = this.streamNode(context, file, plan, at);
node.connect(context.destination);
// Where the plan starts past the head, the first stretch is decoded before the clock runs.
while (feed.busy) await new Promise((resolve) => setTimeout(resolve, 5));
const step = STREAM_RATE / 2;
const hold = (frame: number) => {
if (frame >= frames) return;
void context.suspend(frame / STREAM_RATE).then(async () => {
do await new Promise((resolve) => setTimeout(resolve, 5));
while (feed.busy);
hold(frame + step);
void context.resume();
});
};
hold(step);
const rendered = await context.startRendering();
const skipped = feed.skipped;
source.stop();
return {
channels: Array.from({ length: rendered.numberOfChannels }, (_, c) => rendered.getChannelData(c)),
skipped,
};
}
/** All of the copy a name streams from, decoded as the feed decodes it (test hook). */
async streamedWhole(name: string): Promise<Float32Array[] | null> {
const file = this.streamed(name);
if (!file || !(await file.ready())) return null;
return decodeSpan(file, 0, file.frames);
}
private dropFeed(feed: StreamFeed): void {
if (!this.feeds.delete(feed)) return;
this.skippedBefore += feed.skipped;
}
/**
* A Chip (#329): a worklet of its own that ends with the note. A Pitch on its way moves its
* frequency itself. Without the worklet the way is silent and ends at once.
@@ -2384,6 +2664,24 @@ function loopIn(buffer: AudioBuffer, start: number, end: number): { loopStart: n
return from < to ? { loopStart: from, loopEnd: to } : { loopStart: 0, loopEnd: last };
}
/** The loudest sample of each of `PEAK_BUCKETS` equal stretches of what is heard of `whole`, at most 1. */
function peaksOf(
whole: AudioBuffer,
{ from, to, reverse }: Pick<CutRange, 'from' | 'to' | 'reverse'>,
): Float32Array {
const peaks = new Float32Array(PEAK_BUCKETS);
for (let c = 0; c < whole.numberOfChannels; c++) {
const data = whole.getChannelData(c);
for (let i = from; i < to; i++) {
const heard = reverse ? to - 1 - i : i - from;
const bucket = Math.floor((heard / (to - from)) * PEAK_BUCKETS);
peaks[bucket] = Math.max(peaks[bucket]!, Math.abs(data[i]!));
}
}
for (let b = 0; b < PEAK_BUCKETS; b++) peaks[b] = Math.min(1, peaks[b]!);
return peaks;
}
/** The loudest sample of each side over the stretch a meter looks back over, 1 at full scale. */
function readMeter(meter: BusMeter): [number, number] {
const [left, right] = meter.sides.map((side) => {
src/viewport/fxWorklet.ts
+46 -1
View File
@@ -3,7 +3,8 @@
* plays one sound through a `PitchPlayer`, its pitch the AudioParam `pitch` in semitones, so it
* glides, ramps and eases like any other knob. A Bitcrusher (#322) runs a `Crusher`, a Chip
* (#329) a `ChipPlayer`, a Phaser and a Flanger (#341) theirs, the pitch of a bus (#346) a
* `PitchShifter`, a Limiter (#326) a `Limiter`, a Tremolo and a Vibrato (#320) theirs. Loaded into
* `PitchShifter`, a Limiter (#326) a `Limiter`, a Tremolo and a Vibrato (#320) theirs, a streamed
* sound (#349) a `StreamQueue`. Loaded into
* the context once by the sound library; nothing on the page imports it.
*/
import { type ChipShape, ChipPlayer } from '../core/soundfx/chip';
@@ -12,6 +13,7 @@ import { Limiter, type LimiterNumbers } from '../core/soundfx/dynamics';
import { LFO_SHAPES, type LfoNumbers, Tremolo, Vibrato } from '../core/soundfx/lfo';
import { Flanger, type FlangerNumbers, Phaser, type PhaserNumbers } from '../core/soundfx/phaser';
import { PitchPlayer, PitchShifter } from '../core/soundfx/pitchPlayer';
import { STREAM_RATE, StreamQueue } from '../core/music/streamQueue';
/** What the audio thread has and the page's types do not. */
declare const sampleRate: number;
@@ -167,6 +169,49 @@ class ShiftProcessor extends Effect {
registerProcessor('pitch-shifter', ShiftProcessor);
export interface StreamOptions {
channels: number;
/** The second of the audio clock its first sample is heard at; before it, silence. */
startAt: number;
/** What it starts with, so it is there for the first sample; the rest comes by message. */
first: Float32Array[];
}
/** Samples of the stream read between two reports to the page. */
const STREAM_REPORT = 4800;
/**
* A streamed sound (#349): plays the samples the page sends it, in order, and tells it how far it has
* read so it can send more. 'end' says nothing more comes.
*/
class StreamProcessor extends Source {
private readonly queue: StreamQueue;
private reported = 0;
constructor({ processorOptions }: { processorOptions: StreamOptions }) {
super(processorOptions.startAt);
this.queue = new StreamQueue(processorOptions.channels, STREAM_RATE / sampleRate);
this.queue.push(processorOptions.first);
this.port.onmessage = (event: MessageEvent) => {
const data: unknown = event.data;
if (Array.isArray(data)) this.queue.push(data as Float32Array[]);
else if (data === 'end') this.queue.end();
else this.stop();
};
}
protected render(out: Float32Array[]): boolean {
const going = this.queue.render(out);
if (this.queue.used - this.reported >= STREAM_REPORT) {
this.reported = this.queue.used;
this.port.postMessage({ used: this.queue.used, skipped: this.queue.skipped });
}
return going;
}
}
registerProcessor('stream-player', StreamProcessor);
/** A Bitcrusher (#322). */
class CrushProcessor extends Effect {
private readonly crusher = new Crusher(sampleRate);
src/viewport/projectFiles.ts
+3
View File
@@ -1,6 +1,7 @@
import type { Renderer } from 'pixi.js';
import type { Project } from '../core/model/Project';
import { projectFiles } from '../core/model/projectFiles';
import { streamTable } from '../core/music/streams';
import { requestRender } from '../ui';
import { fontLibrary } from './FontLibrary';
import { installPixelArtPictures } from './pixelArtPicture';
@@ -46,5 +47,7 @@ export function preloadProjectFiles(project: Project): void {
textureLibrary.preload(files.textures);
fontLibrary.preload(files.fonts);
pixelCels.preload(files.cels);
// Which of them stream (#349) first, so their copies are fetched rather than they decoded.
soundLibrary.setStreams(streamTable(project));
soundLibrary.preload(files.sounds);
}
src/viewport/streaming.ts
+376
View File
@@ -0,0 +1,376 @@
/**
* Streaming a sound (#349) on the page's side: the copy is fetched a piece at a time, read page by
* page, and decoded by WebCodecs a few seconds ahead of what is heard; the samples go to a
* `stream-player` on the worklet and are let go there once played. Only the compressed bytes and the
* first seconds of each copy are kept, so a long sound weighs a few megabytes rather than a hundred.
*/
import { OggOpusReader, type PlacedPacket } from '../core/music/ogg';
import { STREAM_RATE } from '../core/music/streamQueue';
import { StreamCursor, type StreamPlan } from '../core/music/streams';
/** Bytes fetched at a time, some 13 s at 160 kbit/s. */
const CHUNK_BYTES = 256 * 1024;
/** Samples of each copy decoded once it is named, so it starts at once (`StreamFile.ready`). */
const HEAD = 2 * STREAM_RATE;
/**
* Samples decoded before a place jumped to and thrown away: measured in Chrome, 32 packets of 20 ms
* make what follows the same to the bit as decoding from the start, 4 come out 0.06 off.
*/
const PREROLL = 32 * 960;
/** Samples kept sent ahead of what the worklet has read, and the most decoded at once. */
const LEAD = 4 * STREAM_RATE;
const BATCH = STREAM_RATE;
/** One copy, shared by every voice streaming it: its bytes as they come, and its first seconds decoded. */
export class StreamFile {
readonly reader = new OggOpusReader();
/** Samples per channel without the pre-skip, as the asset says. */
head: Float32Array[] | null = null;
failed = false;
private fetched = 0;
private complete = false;
private fetching: Promise<void> | null = null;
private heading: Promise<boolean> | null = null;
constructor(
readonly url: string,
readonly frames: number,
) {}
get channels(): number {
return this.reader.channels ?? 2;
}
get preSkip(): number {
return this.reader.preSkip ?? 0;
}
/** Bytes fetched so far (test hook). */
get bytes(): number {
return this.fetched;
}
/** Its head fetched and decoded; false where it cannot be had. */
ready(): Promise<boolean> {
return (this.heading ??= (async () => {
try {
this.head = await decodeSpan(this, 0, Math.min(HEAD, this.frames));
return true;
} catch (error: unknown) {
this.failed = true;
console.error(`[sounds] cannot stream ${this.url}`, error);
return false;
}
})());
}
/** Fetches until the packet holding stream sample `sample` is there, or the copy has ended. */
async reach(sample: number): Promise<void> {
while (!this.complete && this.lastEnd() <= sample) await this.more();
}
/** Every byte of the copy, fetched if it is not yet: what the picture of it is decoded from. */
async whole(): Promise<ArrayBuffer> {
const response = await fetch(this.url);
if (!response.ok) throw new Error(`HTTP ${response.status}`);
return response.arrayBuffer();
}
private lastEnd(): number {
const last = this.reader.packets.at(-1);
return last ? last.at + last.samples : 0;
}
private more(): Promise<void> {
return (this.fetching ??= this.fetchMore().finally(() => {
this.fetching = null;
}));
}
/** The next piece by `Range`; a server that sends the whole file instead is taken at its word. */
private async fetchMore(): Promise<void> {
const response = await fetch(this.url, {
headers: { Range: `bytes=${this.fetched}-${this.fetched + CHUNK_BYTES - 1}` },
});
if (response.status === 416) {
this.complete = true;
return;
}
if (!response.ok) throw new Error(`HTTP ${response.status}`);
const bytes = new Uint8Array(await response.arrayBuffer());
if (response.status === 200 && this.fetched > 0) throw new Error('the server sent all of it again');
this.reader.push(bytes);
this.fetched += bytes.length;
const total = Number(response.headers.get('content-range')?.split('/')[1]);
if (response.status === 200 || bytes.length === 0 || (Number.isFinite(total) && this.fetched >= total))
this.complete = true;
}
}
/**
* Decodes stream samples on one decoder, reading on from where it stopped while it is asked for what
* follows, and settling first wherever it is asked to jump.
*/
class StreamDecoder {
private decoder: AudioDecoder | null = null;
/** The next packet to hand it, and the stream sample the next sample it gives back is at. */
private next = 0;
private outAt = 0;
/** What it has given back and not yet been asked for, from `outAt` back. */
private spare: Float32Array[] = [];
private outputs: Float32Array[][] = [];
private fed = 0;
private given = 0;
private wake: (() => void) | null = null;
private failure: Error | null = null;
constructor(private readonly file: StreamFile) {}
/** The samples of the copy from `from` to `to`, counted without its pre-skip. */
async read(from: number, to: number): Promise<Float32Array[]> {
const { file } = this;
const start = from + file.preSkip;
const end = to + file.preSkip;
const held = () => this.outAt - (this.spare[0]?.length ?? 0);
if (!this.decoder || start !== held()) await this.jump(start);
const spareFrom = held();
const parts: Float32Array[][] = [this.spare];
let have = this.outAt;
this.spare = [];
while (have < end) {
await file.reach(have);
const packet = file.reader.packets[this.next];
if (!packet) break;
this.feed(packet);
const got = await this.take();
parts.push(got);
have += got[0]?.length ?? 0;
}
this.outAt = have;
const joined = join(parts, file.channels, have - spareFrom);
const offset = start - spareFrom;
const kept = joined.map((side) => side.slice(offset, offset + (end - start)));
this.spare = joined.map((side) => side.slice(offset + (end - start)));
return kept;
}
close(): void {
if (this.decoder && this.decoder.state !== 'closed') this.decoder.close();
this.decoder = null;
}
/** A new decoder, from far enough before `sample` that it has settled there; what came before goes. */
private async jump(sample: number): Promise<void> {
this.close();
const { file } = this;
await file.reach(sample);
const packets = file.reader.packets;
let index = 0;
while (index + 1 < packets.length && packets[index + 1]!.at <= sample - PREROLL) index += 1;
this.next = index;
this.outAt = packets[index]?.at ?? 0;
this.spare = [];
this.outputs = [];
this.fed = this.given = 0;
this.failure = null;
const decoder = new AudioDecoder({
output: (data) => {
const sides = Array.from({ length: file.channels }, (_, c) => {
const side = new Float32Array(data.numberOfFrames);
data.copyTo(side, { planeIndex: Math.min(c, data.numberOfChannels - 1), format: 'f32-planar' });
return side;
});
data.close();
this.outputs.push(sides);
this.given += 1;
this.wake?.();
},
error: (error) => {
this.failure = error;
this.wake?.();
},
});
// No description: Chrome would apply the pre-skip by itself, and only to the first packet it gets.
decoder.configure({ codec: 'opus', sampleRate: STREAM_RATE, numberOfChannels: file.channels });
this.decoder = decoder;
// Every packet that ends before where it is asked for settles it and is thrown away; `read` cuts
// the rest off the one it starts in.
while (this.next < packets.length && packets[this.next]!.at + packets[this.next]!.samples <= sample) {
this.feed(packets[this.next]!);
await this.take();
}
this.outAt = packets[this.next]?.at ?? this.outAt;
}
private feed(packet: PlacedPacket): void {
const decoder = this.decoder!;
decoder.decode(
new EncodedAudioChunk({
type: 'key',
timestamp: Math.round((packet.at * 1e6) / STREAM_RATE),
data: packet.data,
}),
);
this.next += 1;
this.fed += 1;
}
/** The samples of the packet handed over last, once the decoder has given them back. */
private async take(): Promise<Float32Array[]> {
while (this.given < this.fed && !this.failure)
await new Promise<void>((resolve) => {
this.wake = resolve;
});
this.wake = null;
if (this.failure) throw this.failure;
const out = this.outputs.splice(0);
return join(
out,
this.file.channels,
out.reduce((sum, part) => sum + (part[0]?.length ?? 0), 0),
);
}
}
/** The samples of a copy from `from` to `to` on a decoder of their own: what a head is. */
export async function decodeSpan(file: StreamFile, from: number, to: number): Promise<Float32Array[]> {
await file.reach(0);
if (file.reader.channels === null) throw new Error('no Opus in it');
const decoder = new StreamDecoder(file);
try {
return await decoder.read(from, to);
} finally {
decoder.close();
}
}
function join(parts: readonly Float32Array[][], channels: number, length: number): Float32Array[] {
return Array.from({ length: channels }, (_, c) => {
const side = new Float32Array(Math.max(0, length));
let at = 0;
for (const part of parts) {
const data = part[c] ?? part[0];
if (!data) continue;
side.set(data.subarray(0, Math.max(0, length - at)), at);
at += data.length;
}
return side;
});
}
/**
* Keeps one `stream-player` fed along its plan: the head where the plan is in it, decoded samples
* after that, never more than `LEAD` ahead of what it has read. Tells `ended` once it has played out.
*/
export class StreamFeed {
private readonly cursor: StreamCursor;
private readonly decoder: StreamDecoder;
private node: AudioWorkletNode | null = null;
private sent = 0;
private used = 0;
private pumping = false;
private stopped = false;
private finished = false;
/** Samples that came too late to be heard, as the worklet last said (test hook). */
skipped = 0;
constructor(
private readonly file: StreamFile,
plan: StreamPlan,
private readonly ended: () => void,
) {
this.cursor = new StreamCursor(plan);
this.decoder = new StreamDecoder(file);
}
/**
* What the player starts with where its plan starts in the head: handed over as it is made, so it
* is there for the first sample heard. Sent by message, it came a few milliseconds late, and a
* sound started at once lost its attack. Empty where the plan starts past the head.
*/
opening(): Float32Array[] {
const at = this.cursor.peek();
const head = this.file.head;
if (at === null || !head || at >= (head[0]?.length ?? 0)) return [];
const span = this.cursor.next(head[0]!.length - at)!;
this.sent += span.to - span.from;
return head.map((side) => side.slice(span.from, span.to));
}
/** Feeds that player from now on. */
attach(node: AudioWorkletNode): void {
this.node = node;
node.port.onmessage = (event: MessageEvent) => {
const data: unknown = event.data;
if (data === 'ended') {
this.close();
this.ended();
} else if (typeof data === 'object' && data !== null && 'used' in data) {
const report = data as { used: number; skipped: number };
this.used = report.used;
this.skipped = report.skipped;
void this.pump();
}
};
void this.pump();
}
stop(): void {
if (this.stopped) return;
this.close();
if (!this.node) return;
this.node.port.postMessage('stop');
this.node.port.close();
this.node.disconnect();
}
/** Whether it is decoding or sending right now: what an offline clock waits for (test hook). */
get busy(): boolean {
return this.pumping;
}
/** Read anew each time: a stop comes in while a stretch is being decoded. */
private halted(): boolean {
return this.stopped;
}
private close(): void {
this.stopped = true;
if (this.node) this.node.port.onmessage = null;
this.decoder.close();
}
private async pump(): Promise<void> {
const node = this.node;
if (!node || this.pumping || this.stopped || this.finished) return;
this.pumping = true;
try {
while (!this.halted() && this.sent - this.used < LEAD) {
const at = this.cursor.peek();
if (at === null) {
this.finished = true;
node.port.postMessage('end');
break;
}
const head = this.file.head;
const inHead = head !== null && at < (head[0]?.length ?? 0);
const span = this.cursor.next(inHead ? head[0]!.length - at : BATCH)!;
const samples = inHead
? head.map((side) => side.slice(span.from, span.to))
: await this.decoder.read(span.from, span.to);
if (this.halted()) break;
node.port.postMessage(
samples,
samples.map((side) => side.buffer),
);
this.sent += span.to - span.from;
}
} catch (error: unknown) {
console.error(`[sounds] streaming ${this.file.url} broke off`, error);
if (!this.halted()) node.port.postMessage('end');
this.finished = true;
} finally {
this.pumping = false;
}
}
}