From 82e96e680f3e5bc6cfa97b34d866ccea1d822f1e Mon Sep 17 00:00:00 2001 From: MelekTaus Date: Fri, 9 Oct 2026 20:53:49 +0200 Subject: [PATCH] Decode a clip from a file of its own, not its whole sound (#349) A clip was cut from its sound's file, so a game decoded the whole file for it and held it for as long as the page was open. A long recording played through a few short clips cost the recording. Every clip now has a copy of its cut as Opus, with Stream on or off, and a game decodes that instead. - StreamKeeper makes the copy of each clip whether its sound streams or not. Stream only decides whether a sound gets a copy of its own. - clipCopyTable says which cut decodes from which copy. The sound library loads the copy for a clip unless the whole file is decoded already. Then the clip is cut from that as before. - projectFiles decodes the whole file ahead only where something may play the sound itself: no clips, a clip without a fitting copy, or the sound's id anywhere in an asset that can name one. That last test walks the data, not the fields, so no place is missed. - A copy that cannot be had falls back to the whole file. A library that bakes keeps to the whole files, as the bake's fingerprint does. Tests: - Gate: CHECK EXIT 0, 280 files, 4319 tests (7 new: streams +4, sound-library +3). - e2e 34: ALL PASSED, 108 checks (2 new). With Stream off a fresh library decodes the ten-second reversed clip's copy and not the five-minute file. It is the cut made from the original to within 0.0091 a sample and 0.013 dB in any 20 ms. Once a node plays the sound itself, the whole file is decoded ahead again. - e2e 65: all passed with this change. - Counter-checks red as they should be: the library ignoring copies, the whole file always decoded ahead, clip copies only with Stream. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- src/core/model/projectFiles.ts | 49 ++++++++++++++++- src/core/music/streams.ts | 33 ++++++++--- src/editor/debug/soundHooks.ts | 48 +++++++++++++++- src/editor/sound/StreamKeeper.ts | 20 +++---- src/viewport/SoundLibrary.ts | 53 +++++++++++++++--- src/viewport/projectFiles.ts | 6 +- tests/core/streams.test.ts | 82 +++++++++++++++++++++++++--- tests/viewport/sound-library.test.ts | 48 ++++++++++++++++ tools/e2e/34-sound.mjs | 33 +++++++++++ 10 files changed, 332 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index a328c947..f5c16e1c 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh The browser suite and the three example games need a machine with a GPU: ```bash -node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 1935 checks (GM_STEPS=20 for one step) +node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 1937 checks (GM_STEPS=20 for one step) # every tool signs in first – see the Accounts page node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs diff --git a/src/core/model/projectFiles.ts b/src/core/model/projectFiles.ts index 353c1115..25ab16ff 100644 --- a/src/core/model/projectFiles.ts +++ b/src/core/model/projectFiles.ts @@ -1,9 +1,12 @@ import { validBake } from '../music/bake'; -import { streamsOf } from '../properties/sounds'; +import { clipCopyTable } from '../music/streams'; +import { cutOf, streamsOf } from '../properties/sounds'; +import type { AssetType } from './AssetTypes'; +import type { SoundAsset } from './Asset'; import { isFontFile } from './font'; import { celFilesOf } from './pixelart'; import type { Project } from './Project'; -import { isSoundFile } from './sound'; +import { clipsOf, cutName, isSoundFile } from './sound'; import { isTextureFile } from './texture'; /** The stored files a project draws and plays, per library – what is loaded before anything asks. */ @@ -20,17 +23,57 @@ export interface ProjectFiles { * an upload that failed – is left out, as the libraries would never get it anyway. So is a bake that * no longer holds (#340): it would never be played. So is a sound that streams (#349): the sound * library fetches the start of its copy instead, and decoded ahead it would weigh what streaming saves. + * A sound only played through its clips has their copies loaded rather than its whole file. */ export function projectFiles(project: Project): ProjectFiles { const files: ProjectFiles = { textures: [], fonts: [], cels: [], sounds: [] }; + const copies = clipCopyTable(project); + let named: ReadonlySet | null = null; + const namedWhole = (asset: SoundAsset) => (named ??= namedStrings(project)).has(asset.id); for (const asset of project.assets) { if (asset.type === 'texture' && isTextureFile(asset.file)) files.textures.push(asset.file); else if (asset.type === 'font' && isFontFile(asset.file)) files.fonts.push(asset.file); else if (asset.type === 'pixelart') files.cels.push(...celFilesOf(asset)); else if (asset.type === 'sound' && isSoundFile(asset.file) && !streamsOf(asset.props)) - files.sounds.push(asset.file); + files.sounds.push(...decodedAhead(asset, copies, namedWhole)); else if ((asset.type === 'pattern' || asset.type === 'song') && validBake(project, asset)) files.sounds.push(asset.bake!.file); } return files; } + +/** + * What of a sound is decoded ahead: its file where something may play it whole – a sound without + * clips, one the project names itself, one with a clip whose copy is not there yet – else the cut of + * each clip, which the sound library plays from its copy. + */ +function decodedAhead( + asset: SoundAsset, + copies: ReadonlyMap, + namedWhole: (asset: SoundAsset) => boolean, +): string[] { + const cuts = clipsOf(asset).map((clip) => cutName(asset.file, cutOf(clip.props))); + const whole = cuts.length === 0 || cuts.some((cut) => !copies.has(cut)) || namedWhole(asset); + return whole ? [asset.file] : [...new Set(cuts)]; +} + +/** What holds no reference to a sound, and holds a lot: not walked for one. */ +const UNNAMING: ReadonlySet = new Set(['sound', 'texture', 'pixelart', 'tilemap', 'font']); + +/** + * Every text the project holds outside the assets that cannot name a sound. Where a sound is played + * whole – a sound field, an item of a song, a Sample, a Reverb's impulse – its id is one of them; a + * clip has an id of its own. Looked at as data, not field by field, so no place is missed: one that + * only looks like a reference loads the file as before. + */ +function namedStrings(project: Project): Set { + const seen = new Set(); + const walk = (value: unknown): void => { + if (typeof value === 'string') seen.add(value); + else if (Array.isArray(value)) value.forEach(walk); + else if (typeof value === 'object' && value !== null && !ArrayBuffer.isView(value)) + Object.values(value).forEach(walk); + }; + for (const asset of project.assets) if (!UNNAMING.has(asset.type)) walk(asset); + return seen; +} diff --git a/src/core/music/streams.ts b/src/core/music/streams.ts index d9a8f97a..04090230 100644 --- a/src/core/music/streams.ts +++ b/src/core/music/streams.ts @@ -1,9 +1,9 @@ /** * Sounds streamed rather than decoded whole (#349). A sound with `stream` on is played from a copy as - * Opus in Ogg, made in the browser, and each of its clips from a copy of its own cut, so no clip needs - * the whole file either. A bake (#340) is Opus already and always streams. What a game plays is still - * named as before – the file, or the cut a clip is – and the table here says which of those names to - * stream from which copy. + * Opus in Ogg, made in the browser. Every clip of every sound has a copy of its own cut, so no clip + * needs the whole file: streamed where its sound streams, decoded where it does not. A bake (#340) is + * Opus already and always streams. What a game plays is still named as before – the file, or the cut + * a clip is – and the tables here say which of those names to play from which copy. */ import type { SoundAsset, SoundClip, SoundStream } from '../model/Asset'; import type { Project } from '../model/Project'; @@ -97,6 +97,23 @@ function heldCopy(stream: SoundStream | undefined, played: string): StreamedFile return stream && stream.from === played ? { file: stream.file, frames: stream.frames } : null; } +/** + * The cut each clip plays, with the copy it is decoded from instead of its sound's whole file – of a + * sound that streams too, for where a stream cannot be had. A clip whose copy does not fit is left out. + */ +export function clipCopyTable(project: Project): Map { + const table = new Map(); + for (const asset of project.assets) { + if (asset.type !== 'sound' || !asset.file) continue; + for (const clip of clipsOf(asset)) { + const played = cutName(asset.file, cutOf(clip.props)); + const copy = played === asset.file ? null : heldCopy(clip.stream, played); + if (copy) table.set(played, copy.file); + } + } + return table; +} + /** A copy to make: of a sound, or of one of its clips, from what it plays now. */ export interface StreamJob { assetId: string; @@ -104,7 +121,7 @@ export interface StreamJob { from: string; } -/** A copy that is to go: its sound no longer streams, or a clip of it has become the whole file. */ +/** A copy that is to go: its sound no longer streams or has no file, or a clip is the whole file. */ export interface StreamDrop { assetId: string; clipId: string | null; @@ -116,16 +133,16 @@ export function streamWork(project: Project): { jobs: StreamJob[]; drops: Stream const drops: StreamDrop[] = []; for (const asset of project.assets) { if (asset.type !== 'sound') continue; - const on = streamsOf(asset.props) && asset.file !== ''; const keep = (owner: SoundAsset | SoundClip, clipId: string | null, from: string | null) => { if (from === null) { if (owner.stream) drops.push({ assetId: asset.id, clipId }); } else if (owner.stream?.from !== from) jobs.push({ assetId: asset.id, clipId, from }); }; - keep(asset, null, on ? asset.file : null); + keep(asset, null, streamsOf(asset.props) && asset.file !== '' ? asset.file : null); + // A clip has its copy whether its sound streams or not. for (const clip of clipsOf(asset)) { const played = cutName(asset.file, cutOf(clip.props)); - keep(clip, clip.id, on && played !== asset.file ? played : null); + keep(clip, clip.id, asset.file !== '' && played !== asset.file ? played : null); } } return { jobs, drops }; diff --git a/src/editor/debug/soundHooks.ts b/src/editor/debug/soundHooks.ts index 4eabc44c..c4db500f 100644 --- a/src/editor/debug/soundHooks.ts +++ b/src/editor/debug/soundHooks.ts @@ -1,7 +1,8 @@ import { findAssetOf } from '../../core/model/Project'; -import { clipsOf, findSound } from '../../core/model/sound'; +import { projectFiles } from '../../core/model/projectFiles'; +import { clipsOf, cutChannels, cutRange, findSound, parseCutName } from '../../core/model/sound'; import { STREAM_RATE } from '../../core/music/streamQueue'; -import { playedPlan, streamTable } from '../../core/music/streams'; +import { clipCopyTable, playedPlan, streamTable } from '../../core/music/streams'; import { playedSoundOf } from '../../core/properties/playedSound'; import { soundLibrary } from '../../viewport/SoundLibrary'; import { centreOf, type HookContext } from './hookHelpers'; @@ -121,6 +122,49 @@ export function soundHooks({ state, editor }: HookContext) { skipped: played.skipped, }; }, + /** + * What a game decodes that has just opened the project, in a library of its own so nothing this + * page holds counts: the names it decodes ahead, whether that decoded the whole file of a name's + * sound, and what the name then plays held against its cut made from that file – samples, rate, + * the loudest difference of a sample and of the level of any 20 ms. + */ + fresh: async (name: string) => { + const library = soundLibrary.aside(); + library.setStreams(streamTable(state.project)); + library.setCopies(clipCopyTable(state.project)); + const ahead = projectFiles(state.project).sounds; + await Promise.all(ahead.map((file) => library.load(file))); + const { file, cut } = parseCutName(name); + const whole = library.decoded(file) !== null; + const played = await library.load(name); + const original = await soundLibrary.decodeApart(file, STREAM_RATE); + if (!played || !cut || played.sampleRate !== STREAM_RATE) return { ahead, whole, played: null }; + const [reference] = cutChannels( + [original.getChannelData(0)], + cutRange(original.length, STREAM_RATE, cut), + ); + const heard = played.getChannelData(0); + let worst = 0; + let level = 0; + const span = Math.round(0.02 * STREAM_RATE); + for (let from = 0; from + span <= heard.length; from += span) { + let own = 0; + let wanted = 0; + for (let i = from; i < from + span; i++) { + worst = Math.max(worst, Math.abs(heard[i]! - (reference![i] ?? 0))); + own += heard[i]! ** 2; + wanted += (reference![i] ?? 0) ** 2; + } + // Only where it is heard: the fades start from silence. + if (wanted / span > 1e-6) level = Math.max(level, Math.abs(10 * Math.log10(own / wanted))); + } + return { + ahead, + whole, + played: { length: played.length, wanted: reference!.length, worst, level }, + decoded: library.stats().decoded, + }; + }, /** What a sound and each of its clips keep of their copies. */ copies: (id: string) => { const asset = findAssetOf(state.project, id, 'sound'); diff --git a/src/editor/sound/StreamKeeper.ts b/src/editor/sound/StreamKeeper.ts index dc0076ce..3975edad 100644 --- a/src/editor/sound/StreamKeeper.ts +++ b/src/editor/sound/StreamKeeper.ts @@ -14,10 +14,10 @@ import type { EditorState } from '../state'; const SETTLE_MS = 600; /** - * Makes the copies streamed sounds are played from (#349). A moment after the project last changed, - * each sound with `stream` on gets a copy of its file as Opus, and each of its clips one of its cut, - * one at a time, from the file decoded for it and let go again. A copy no longer wanted is taken off - * its sound and deleted once the project has been saved without it. Not while the game runs. + * Makes the copies sounds and clips are played from (#349). A moment after the project last changed, + * each sound with `stream` on gets a copy of its file as Opus, and each clip of any sound one of its + * cut, one at a time, from the file decoded for it and let go again. A copy no longer wanted is taken + * off its sound and deleted once the project has been saved without it. Not while the game runs. */ export class StreamKeeper { private job: { job: StreamJob; share: number } | null = null; @@ -77,7 +77,7 @@ export class StreamKeeper { if (error instanceof BakeCancelled) return this.later(0); this.failed.add(keyOf(next)); this.state.setNotice( - `No stream copy of ${this.label(next)}: ${error instanceof Error ? error.message : String(error)}`, + `No copy of ${this.label(next)}: ${error instanceof Error ? error.message : String(error)}`, 'error', ); } finally { @@ -100,7 +100,7 @@ export class StreamKeeper { const before = Math.round(running.share * 100); running.share = share; const now = Math.round(share * 100); - if (now !== before) this.state.setNotice(`Making a stream copy of ${label}: ${now}%`, 'quiet'); + if (now !== before) this.state.setNotice(`Making a copy of ${label}: ${now}%`, 'quiet'); }, cancelled: () => !this.stillWanted(job), }); @@ -119,14 +119,14 @@ export class StreamKeeper { bytes: uploaded.bytes, }); if (old && old !== uploaded.file && !this.named(old)) dropWhenSaved(this.state, this.sounds, old); - this.state.setNotice(`Made a stream copy of ${label}`, 'quiet'); + this.state.setNotice(`Made a copy of ${label}`, 'quiet'); } - /** Whether the sound still streams and still plays what the copy is being made of. */ + /** Whether the sound, or its clip, still plays what the copy is being made of – a sound still streams. */ private stillWanted(job: StreamJob): boolean { const asset = this.sound(job.assetId); - if (!asset || !streamsOf(asset.props)) return false; - if (job.clipId === null) return asset.file === job.from; + if (!asset) return false; + if (job.clipId === null) return streamsOf(asset.props) && asset.file === job.from; const clip = clipsOf(asset).find((c) => c.id === job.clipId); return clip !== undefined && cutName(asset.file, cutOf(clip.props)) === job.from; } diff --git a/src/viewport/SoundLibrary.ts b/src/viewport/SoundLibrary.ts index c8b7ad6b..8a2d71f8 100644 --- a/src/viewport/SoundLibrary.ts +++ b/src/viewport/SoundLibrary.ts @@ -378,6 +378,8 @@ export class SoundLibrary implements AudioSink { private readonly mixVolumes = new Map(); /** The names streamed (#349) and the copies they stream from, by name and by copy. */ private streams: ReadonlyMap = new Map(); + /** The cut of each clip, with the copy it is decoded from rather than its whole file (#349). */ + private copies: ReadonlyMap = new Map(); private readonly streamFiles = new Map(); private readonly feeds = new Set(); /** Every sample streamed so far that came too late to be heard, of the feeds gone (test hook). */ @@ -398,6 +400,21 @@ export class SoundLibrary implements AudioSink { for (const name of table.keys()) void this.streamed(name)?.ready(); } + /** + * The copies clips are decoded from (#349), each under the cut it plays (`clipCopyTable`). A clip + * whose sound's whole file is decoded anyway is cut from that as before. One that bakes keeps to the + * whole files, as the fingerprint of a bake does. + */ + setCopies(table: ReadonlyMap): void { + this.copies = table; + } + + /** The copy a clip is decoded from, while it has one that has not failed. */ + private copyOf(name: string): string | null { + const copy = this.render ? undefined : this.copies.get(name); + return copy && !this.failed.has(copy) ? copy : null; + } + /** The copy a name streams from, while it streams and the copy has not failed. */ private streamed(name: string): StreamFile | null { const entry = this.streams.get(name); @@ -1755,10 +1772,14 @@ export class SoundLibrary implements AudioSink { /** * Fetches and decodes once per file; a second play while it is in flight waits on the first. A cut - * (#343) loads its file and is made from it. + * (#343) loads its file and is made from it – the cut of a clip with a copy (#349) loads that + * instead, unless the whole file is there already. */ async load(file: string): Promise { if (parseCutName(file).cut) { + const copy = this.copyOf(file); + if (copy && !this.buffers.has(parseCutName(file).file) && (await this.load(copy))) + return this.buffer(file); const whole = await this.load(parseCutName(file).file); return whole ? this.buffer(file) : null; } @@ -2331,6 +2352,13 @@ export class SoundLibrary implements AudioSink { }; } + /** A library of its own, served from where this one is, that has decoded nothing yet (test hook). */ + aside(): SoundLibrary { + const library = new SoundLibrary(this.guarded); + library.configure(this.base); + return library; + } + /** All of the copy a name streams from, decoded as the feed decodes it (test hook). */ async streamedWhole(name: string): Promise { const file = this.streamed(name); @@ -2446,10 +2474,16 @@ export class SoundLibrary implements AudioSink { }); } - /** What a name plays, a cut made from its file the first time; null while the file is not loaded. */ + /** + * What a name plays: the copy of a clip (#349) once it is decoded, else a cut made from its file the + * first time; null while neither is loaded. + */ private buffer(name: string): AudioBuffer | null { const whole = this.buffers.get(name); if (whole) return whole; + const copy = this.copyOf(name); + const copied = copy ? this.buffers.get(copy) : undefined; + if (copied) return copied; const held = this.cuts.get(name); if (held) { this.cuts.delete(name); @@ -2476,12 +2510,17 @@ export class SoundLibrary implements AudioSink { return made; } - /** The samples of its loaded file a name plays: all of them, or the stretch a cut keeps. */ + /** + * The samples of its loaded file a name plays: all of them, or the stretch a cut keeps – all of a + * clip's copy (#349), which is that stretch already. + */ private rangeOf(name: string): (CutRange & { whole: AudioBuffer }) | null { const { file, cut } = parseCutName(name); - const whole = this.buffers.get(file); + const copy = this.copyOf(name); + const copied = copy ? this.buffers.get(copy) : undefined; + const whole = copied ?? this.buffers.get(file); if (!whole) return null; - if (!cut) + if (!cut || copied) return { whole, from: 0, @@ -2498,9 +2537,9 @@ export class SoundLibrary implements AudioSink { /** How long a sound a Reverb takes as its impulse response rings (#328): all it may, while unknown. */ private readonly soundLength = (file: string): number => this.duration(file) ?? REVERB_MAX_DECAY; - /** Whether the file of a name could not be had: a cut of it cannot either. */ + /** Whether the file of a name could not be had: a cut of it cannot either, unless it has a copy. */ private hasFailed(name: string): boolean { - return this.failed.has(parseCutName(name).file); + return this.copyOf(name) === null && this.failed.has(parseCutName(name).file); } /** Loads the FX worklet (Pitch, Bitcrusher, Chip, Phaser, Flanger) into the context once; says whether it is there. */ diff --git a/src/viewport/projectFiles.ts b/src/viewport/projectFiles.ts index 59983582..85d32ff0 100644 --- a/src/viewport/projectFiles.ts +++ b/src/viewport/projectFiles.ts @@ -1,7 +1,7 @@ import type { Renderer } from 'pixi.js'; import type { Project } from '../core/model/Project'; import { projectFiles } from '../core/model/projectFiles'; -import { streamTable } from '../core/music/streams'; +import { clipCopyTable, streamTable } from '../core/music/streams'; import { requestRender } from '../ui'; import { fontLibrary } from './FontLibrary'; import { installPixelArtPictures } from './pixelArtPicture'; @@ -47,7 +47,9 @@ export function preloadProjectFiles(project: Project): void { textureLibrary.preload(files.textures); fontLibrary.preload(files.fonts); pixelCels.preload(files.cels); - // Which of them stream (#349) first, so their copies are fetched rather than they decoded. + // Which of them stream (#349) first, so their copies are fetched rather than they decoded, and + // which clips decode from a copy of their own rather than from their sound's whole file. soundLibrary.setStreams(streamTable(project)); + soundLibrary.setCopies(clipCopyTable(project)); soundLibrary.preload(files.sounds); } diff --git a/tests/core/streams.test.ts b/tests/core/streams.test.ts index 60c01502..ae2d4475 100644 --- a/tests/core/streams.test.ts +++ b/tests/core/streams.test.ts @@ -10,11 +10,14 @@ import { SetAssetPropertyCommand, } from '../../src/core/commands/assetCommands'; import { AddClipCommand, SetClipPropsCommand } from '../../src/core/commands/clipCommands'; +import { SetSongCommand } from '../../src/core/commands/songCommands'; import type { SongAsset, SoundAsset, SoundStream } from '../../src/core/model/Asset'; import { projectFiles } from '../../src/core/model/projectFiles'; import { cutName } from '../../src/core/model/sound'; +import { newItem } from '../../src/core/music/song'; import { STREAM_RATE } from '../../src/core/music/streamQueue'; import { + clipCopyTable, heldStream, isStreamUpload, playedPlan, @@ -73,9 +76,12 @@ describe('which sounds stream', () => { expect([...streamTable(f.project).keys()]).toEqual([FILE]); }); - it('asks for a copy of the sound and of each clip with a cut of its own, and lets them go once it is off', () => { + it('asks for a copy of each clip with a cut of its own, of the sound once it streams, and lets that go once it is off', () => { const f = streamed(); - expect(streamWork(f.project)).toEqual({ jobs: [], drops: [] }); + expect(streamWork(f.project)).toEqual({ + jobs: [{ assetId: f.sound.id, clipId: f.clipId, from: f.cut() }], + drops: [], + }); f.turn(true); expect(streamWork(f.project).jobs).toEqual([ { assetId: f.sound.id, clipId: null, from: FILE }, @@ -86,14 +92,16 @@ describe('which sounds stream', () => { expect(streamWork(f.project)).toEqual({ jobs: [], drops: [] }); f.stack.execute(new SetClipPropsCommand(f.sound.id, f.clipId, { reverse: true })); expect(streamWork(f.project).jobs).toEqual([{ assetId: f.sound.id, clipId: f.clipId, from: f.cut() }]); + f.sound.clips![0]!.stream = f.copy(f.cut(), CLIP_COPY); f.turn(false); - expect(streamWork(f.project)).toEqual({ - jobs: [], - drops: [ - { assetId: f.sound.id, clipId: null }, - { assetId: f.sound.id, clipId: f.clipId }, - ], - }); + expect(streamWork(f.project)).toEqual({ jobs: [], drops: [{ assetId: f.sound.id, clipId: null }] }); + }); + + it('lets the copy of a clip go once it plays the whole file', () => { + const f = streamed(); + f.sound.clips![0]!.stream = f.copy(f.cut(), CLIP_COPY); + f.stack.execute(new SetClipPropsCommand(f.sound.id, f.clipId, { start: 0, end: 0 })); + expect(streamWork(f.project)).toEqual({ jobs: [], drops: [{ assetId: f.sound.id, clipId: f.clipId }] }); }); it('leaves a sound that streams out of what is decoded ahead, its copy made or not', () => { @@ -121,6 +129,62 @@ describe('which sounds stream', () => { }); }); +describe('a clip decoded from its own copy', () => { + /** A sound only played through one clip of it, by a song. */ + function clipped() { + const f = makeSpriteScene('Box'); + const made = new CreateSoundCommand('Music', FILE); + f.stack.execute(made); + const sound = made.asset; + const clip = new AddClipCommand(sound.id, { start: 1, end: 3 }); + f.stack.execute(clip); + const song = new CreateAssetCommand('song', 'Theme'); + f.stack.execute(song); + const track = (song.asset as SongAsset).tracks[0]!.id; + const play = (asset: string) => + f.stack.execute( + new SetSongCommand(song.asset.id, { + items: [newItem({ track, kind: 'sound', asset, tick: 0, length: 96 })], + }), + ); + play(clip.clipId); + const cut = () => cutName(FILE, cutOf(sound.clips![0]!.props)); + const copy = () => + (sound.clips![0]!.stream = { file: CLIP_COPY, from: cut(), frames: 96000, bytes: 4000 }); + return { ...f, sound, clipId: clip.clipId, play, cut, copy }; + } + + it('decodes the copy of each clip whose copy fits, whether its sound streams or not', () => { + const f = clipped(); + expect(clipCopyTable(f.project).size).toBe(0); + f.copy(); + expect([...clipCopyTable(f.project)]).toEqual([[f.cut(), CLIP_COPY]]); + f.stack.execute(new SetAssetPropertyCommand(f.sound.id, 'stream', true)); + expect([...clipCopyTable(f.project)]).toEqual([[f.cut(), CLIP_COPY]]); + f.stack.execute(new SetClipPropsCommand(f.sound.id, f.clipId, { fadeIn: 0.5 })); + expect(clipCopyTable(f.project).size).toBe(0); + }); + + it('loads the copies of its clips ahead rather than the whole file, until something plays it whole', () => { + const f = clipped(); + expect(projectFiles(f.project).sounds).toEqual([FILE]); + f.copy(); + expect(projectFiles(f.project).sounds).toEqual([f.cut()]); + f.play(f.sound.id); + expect(projectFiles(f.project).sounds).toEqual([FILE]); + }); + + it('loads the whole file while a clip has no copy of its cut, or is the whole file', () => { + const f = clipped(); + f.copy(); + const whole = new AddClipCommand(f.sound.id, { start: 0, end: 0 }); + f.stack.execute(whole); + expect(projectFiles(f.project).sounds).toEqual([FILE]); + f.stack.execute(new SetClipPropsCommand(f.sound.id, whole.clipId, { start: 0.5 })); + expect(projectFiles(f.project).sounds).toEqual([FILE]); + }); +}); + describe('walking a copy', () => { it('plays a sound from where it is asked to, round its whole length where it loops', () => { expect(playedPlan(1000, 0, false)).toEqual({ from: 0, until: 1000, loop: null }); diff --git a/tests/viewport/sound-library.test.ts b/tests/viewport/sound-library.test.ts index 2b367449..607919c1 100644 --- a/tests/viewport/sound-library.test.ts +++ b/tests/viewport/sound-library.test.ts @@ -505,6 +505,54 @@ describe('the sound library', () => { expect(library.stats().playing).toBe(1); }); + describe('a clip with a copy of its own (#349)', () => { + const cut = `${FILE}#0.5~1~0.005~0.005~linear~linear`; + const copy = `${'a1'.repeat(6)}/${'d4'.repeat(32)}.ogg`; + + /** Which files are fetched; one ending in `missing` is not there. */ + function fetching(missing = '#'): string[] { + const urls: string[] = []; + vi.stubGlobal('fetch', (url: string) => { + urls.push(url.replace(/^.*\/sounds\//, '')); + if (url.endsWith(missing)) return Promise.resolve({ ok: false, status: 404 }); + return Promise.resolve({ ok: true, arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)) }); + }); + return urls; + } + + it('decodes its copy and plays all of it, the whole file never fetched', async () => { + const urls = fetching(); + const library = new SoundLibrary(false); + library.setCopies(new Map([[cut, copy]])); + await play(library, cut); + expect(urls).toEqual([copy]); + expect(library.decoded(FILE)).toBeNull(); + // The copy is the cut already: played as it is, not cut again. + expect((sources[0]?.buffer as { length: number }).length).toBe(1200); + expect(library.duration(cut)).toBe(1.5); + expect(library.stats()).toMatchObject({ started: 1, decoded: 1 }); + }); + + it('is cut from the whole file as before while that is decoded anyway', async () => { + const urls = fetching(); + const library = new SoundLibrary(false); + library.setCopies(new Map([[cut, copy]])); + await play(library); + await play(library, cut); + expect(urls).toEqual([FILE]); + expect((sources[1]?.buffer as { length: number }).length).toBe(400); + }); + + it('is cut from the whole file when its copy cannot be had', async () => { + const urls = fetching('.ogg'); + const library = new SoundLibrary(false); + library.setCopies(new Map([[cut, copy]])); + await play(library, cut); + expect(urls).toEqual([copy, FILE]); + expect((sources[0]?.buffer as { length: number }).length).toBe(400); + }); + }); + it('sends the master out through a limiter at -1 dBFS once the worklet is there, late by it (#359)', async () => { let load: () => void = () => {}; loadModule = () => new Promise((resolve) => (load = resolve)); diff --git a/tools/e2e/34-sound.mjs b/tools/e2e/34-sound.mjs index 9f6e28bf..8724167c 100644 --- a/tools/e2e/34-sound.mjs +++ b/tools/e2e/34-sound.mjs @@ -2763,6 +2763,39 @@ export async function sound(t) { JSON.stringify(exact), ); + // 34h-5. With Stream off the clip keeps its copy, and a game only playing the sound through it + // decodes the copy, not the five minutes. The counter-proof plays the sound itself: then the + // whole file is decoded ahead again. + await mutate(`__gm.state.project.assets.find((a) => a.id === 'sn-long').props.stream = false;`); + await evaluate('__gm.streams.idle()'); + const keptCopies = await json("__gm.streams.copies('sn-long')"); + const freshly = async () => + JSON.parse(await evaluate(`__gm.streams.fresh(${JSON.stringify(clipName)}).then(JSON.stringify)`)); + const fresh = await freshly(); + check( + 'with Stream off the clip keeps a copy of its cut, and a game decodes that rather than the whole file, as the cut sounds', + keptCopies?.sound === null && + keptCopies.clips[0]?.stream?.from === clipName && + fresh?.ahead.includes(clipName) && + !fresh.ahead.includes(long.file) && + fresh.whole === false && + fresh.decoded === fresh.ahead.length && + fresh.played?.length === 10 * 48000 && + fresh.played.wanted === fresh.played.length && + fresh.played.level < 0.5 && + fresh.played.worst < 0.1, + JSON.stringify({ keptCopies, fresh }), + ); + await startWith(playsLong); + const freshWhole = await freshly(); + check( + 'once the game plays the sound itself, its whole file is decoded ahead again', + freshWhole?.ahead.includes(long.file) && + !freshWhole.ahead.includes(clipName) && + freshWhole.whole === true, + JSON.stringify({ ahead: freshWhole?.ahead, whole: freshWhole?.whole }), + ); + await mutate(` const p = __gm.state.project; p.assets = p.assets.filter((a) => !['sn-long', 'sn-bus', ${JSON.stringify(longSong.id)}].includes(a.id));