diff --git a/src/core/model/projectFiles.ts b/src/core/model/projectFiles.ts index a6906e9f..aa49e5a2 100644 --- a/src/core/model/projectFiles.ts +++ b/src/core/model/projectFiles.ts @@ -1,13 +1,15 @@ import { isFontFile } from './font'; import { celFilesOf } from './pixelart'; import type { Project } from './Project'; +import { isSoundFile } from './sound'; import { isTextureFile } from './texture'; -/** The stored files a project draws with, per library – what is loaded before anything asks. */ +/** The stored files a project draws and plays, per library – what is loaded before anything asks. */ export interface ProjectFiles { textures: string[]; fonts: string[]; cels: string[]; + sounds: string[]; } /** @@ -16,11 +18,12 @@ export interface ProjectFiles { * an upload that failed – is left out, as the libraries would never get it anyway. */ export function projectFiles(project: Project): ProjectFiles { - const files: ProjectFiles = { textures: [], fonts: [], cels: [] }; + const files: ProjectFiles = { textures: [], fonts: [], cels: [], sounds: [] }; for (const asset of project.assets) { if (asset.type === 'texture' && isTextureFile(asset.file)) files.textures.push(asset.file); else if (asset.type === 'font' && isFontFile(asset.file)) files.fonts.push(asset.file); else if (asset.type === 'pixelart') files.cels.push(...celFilesOf(asset)); + else if (asset.type === 'sound' && isSoundFile(asset.file)) files.sounds.push(asset.file); } return files; } diff --git a/src/viewport/SoundLibrary.ts b/src/viewport/SoundLibrary.ts index 0cbe41b3..b1c3485b 100644 --- a/src/viewport/SoundLibrary.ts +++ b/src/viewport/SoundLibrary.ts @@ -14,6 +14,8 @@ const DEFAULT_BASE = '/game-maker/data/sounds'; const SHIFTED_KEPT = 32; /** Seconds a changed parameter takes to get most of the way there: quick, but no click (#315). */ const GLIDE = 0.03; +/** The rate a sound loaded ahead of its first play is decoded at; a source resamples it on the way out. */ +const DECODE_RATE = 48000; /** * One number of a voice that can still change while it plays (#315): a level, a share of a Mix, the @@ -68,11 +70,13 @@ interface Pending { * * The context is created on the first play, not at start-up: a browser refuses one before the page * has been interacted with, and an editor that was only ever looked at should not be holding an - * audio device open. + * audio device open. What `preload` fetches before that is decoded by an offline context, which + * holds no device. */ export class SoundLibrary implements AudioSink { private base = DEFAULT_BASE; private context: AudioContext | null = null; + private offline: OfflineAudioContext | null = null; private readonly buffers = new Map(); /** Pitched copies of decoded sounds, oldest use first (see `pitched`). */ private readonly shifted = new Map(); @@ -212,6 +216,14 @@ export class SoundLibrary implements AudioSink { return this.buffers.get(file)?.duration ?? null; } + /** + * Starts loading every file not held yet. Waiting for the first play left that play late: its + * start was not heard, and a graph tuning it in the meantime had moved on by the time it started. + */ + preload(files: Iterable): void { + for (const file of files) if (file) void this.load(file); + } + /** Fetches and decodes once per file; a second play while it is in flight waits on the first. */ async load(file: string): Promise { const held = this.buffers.get(file); @@ -224,7 +236,7 @@ export class SoundLibrary implements AudioSink { const response = await fetch(`${this.base}/${file}`); if (!response.ok) throw new Error(`HTTP ${response.status}`); const bytes = await response.arrayBuffer(); - const buffer = await this.audio().decodeAudioData(bytes); + const buffer = await this.decoder().decodeAudioData(bytes); this.buffers.set(file, buffer); return buffer; } catch (error: unknown) { @@ -371,6 +383,12 @@ export class SoundLibrary implements AudioSink { return (this.context ??= new AudioContext()); } + /** The context that is already playing, or else one that decodes without opening a device. */ + private decoder(): BaseAudioContext { + if (this.context || typeof OfflineAudioContext === 'undefined') return this.audio(); + return (this.offline ??= new OfflineAudioContext(1, 1, DECODE_RATE)); + } + /** Everything that would have silenced this file for this owner, counted rather than flagged. */ private cancelsFor(file: string, owner: string): number { return (this.stops.get(file) ?? 0) + this.allStops + (this.ownerStops.get(owner) ?? 0); diff --git a/src/viewport/projectFiles.ts b/src/viewport/projectFiles.ts index dfb774a0..12ce59a9 100644 --- a/src/viewport/projectFiles.ts +++ b/src/viewport/projectFiles.ts @@ -46,4 +46,5 @@ export function preloadProjectFiles(project: Project): void { textureLibrary.preload(files.textures); fontLibrary.preload(files.fonts); pixelCels.preload(files.cels); + soundLibrary.preload(files.sounds); } diff --git a/tests/core/project-files.test.ts b/tests/core/project-files.test.ts index 3fa8b940..aab12dda 100644 --- a/tests/core/project-files.test.ts +++ b/tests/core/project-files.test.ts @@ -1,6 +1,7 @@ /** * What the editor loads ahead of the first draw: every stored file a project's assets name, so an - * animation or an animated tile does not blink out the first time it reaches a frame of its own. + * animation or an animated tile does not blink out the first time it reaches a frame of its own, + * and the first play of a sound is not late. */ import { describe, expect, it } from 'vitest'; import type { Asset } from '../../src/core/model/Asset'; @@ -12,6 +13,7 @@ const PNG_B = `${'b'.repeat(64)}.webp`; const FONT = `${'c'.repeat(64)}.woff2`; const CEL_A = `${'d'.repeat(64)}.cel`; const CEL_B = `${'e'.repeat(64)}.cel`; +const SOUND = `${'f'.repeat(64)}.ogg`; function projectWith(assets: unknown[]) { const project = createEmptyProject('p', 'P'); @@ -20,7 +22,7 @@ function projectWith(assets: unknown[]) { } describe('the files a project names', () => { - it('lists every texture, font and cel, drawn somewhere or not', () => { + it('lists every texture, font, cel and sound, used somewhere or not', () => { const project = projectWith([ { id: 't1', @@ -57,15 +59,22 @@ describe('the files a project names', () => { { id: 'f3', cels: { l: CEL_A } }, ], }, + { id: 's', type: 'sound', name: 'Blip', props: {}, file: SOUND }, ]); - expect(projectFiles(project)).toEqual({ textures: [PNG_A, PNG_B], fonts: [FONT], cels: [CEL_A, CEL_B] }); + expect(projectFiles(project)).toEqual({ + textures: [PNG_A, PNG_B], + fonts: [FONT], + cels: [CEL_A, CEL_B], + sounds: [SOUND], + }); }); it('leaves out a name that is no stored file', () => { const project = projectWith([ { id: 't', type: 'texture', name: 'Failed', props: {}, file: '', width: 0, height: 0, frames: [] }, { id: 'f', type: 'font', name: 'Failed', props: {}, file: 'x.woff2' }, + { id: 's', type: 'sound', name: 'Failed', props: {}, file: '' }, ]); - expect(projectFiles(project)).toEqual({ textures: [], fonts: [], cels: [] }); + expect(projectFiles(project)).toEqual({ textures: [], fonts: [], cels: [], sounds: [] }); }); }); diff --git a/tests/viewport/sound-library.test.ts b/tests/viewport/sound-library.test.ts index 68e9ab7c..1a14cede 100644 --- a/tests/viewport/sound-library.test.ts +++ b/tests/viewport/sound-library.test.ts @@ -194,6 +194,39 @@ describe('the sound library', () => { expect(sources.filter((s) => s.started)).toHaveLength(2); }); + it('loads ahead of the first play without opening the device, so that play starts at once', async () => { + const opened: string[] = []; + vi.stubGlobal( + 'AudioContext', + class { + constructor() { + opened.push('device'); + return fakeContext(); + } + }, + ); + vi.stubGlobal( + 'OfflineAudioContext', + class { + constructor() { + opened.push('offline'); + return fakeContext(); + } + }, + ); + const library = new SoundLibrary(); + library.preload([FILE, OTHER, '']); + await settle(); + await arrive(); + expect(opened).toEqual(['offline']); + expect(library.stats()).toMatchObject({ decoded: 2, started: 0 }); + // No await: the voice runs in the frame that asked for it, not once the bytes are in. + const voice = library.play(FILE, 1, false); + expect(library.stats()).toMatchObject({ started: 1, playing: 1 }); + expect(library.tune(voice, {})).toBe(true); + expect(opened).toEqual(['offline', 'device']); + }); + it('remembers what it last started, so a muted check can see it at all', async () => { const library = new SoundLibrary(); library.play(FILE, 0.25, true);