Cut a sound to a stretch of its file in its own Sound tab (#343)
A sound asset now opens in a tab that shows its file as a waveform. Two handles set where the sound starts and ends in the file, and that stretch is the sample: Play sound, a song item and an instrument's Sample all play it and nothing else. The file itself is never changed. - The asset has three new rows: Start (s), End (s) and Edge fade (ms). End 0 is the end of the file, the fade is 10 ms by default. Older assets have no values and play the whole file as before. - Every way of playing asks playedSoundOf() for a name instead of the file: the file, or <file>#<start>~<end>~<fade> for a cut. - SoundLibrary makes the cut from the decoded file the first time that name plays and ramps each edge inside the file over the fade. Length and peaks of a cut are read from the file without a copy. - Cut buffers can be made again at any time, so the library lets the oldest go past 64 Mi samples. A voice holds its buffers itself. - The tab draws min/max per pixel from 256-sample block peaks, and the samples as a line when zoomed in far. Wheel zooms, the middle button pans, as in the music tabs. Wave, cut and ruler sit under a mask, so a handle panned out of view does not draw over the panels beside. - Dragging a handle moves that edge, dragging elsewhere spans a new stretch. One drag is one undo step for both edges. Whole file puts the handles back, Play or Space plays the stretch with a playhead. - The inspector preview and the Sound FX preview play the cut too, and the song lanes draw and repeat it. Tests: - Unit: sounds 20, wave-view 8 (new), sound-library 68, plus 258 in the touched areas (instrument, song, pattern, inspector, tabs) - e2e 34 on melektaus-atx: ALL PASSED, 57 checks (7 new) - Gate on melektaus-atx: CHECK EXIT 0, 253 files, 3936 tests Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2fdf97f7de
commit
f1f4a57589
24 files changed
+1442
-53
No files matched your search
+101
-15
@@ -21,6 +21,7 @@ import { type BusLevel, Playbacks, type SequencePlay } from '../core/music/playb
|
||||
import type { ScheduledClip, ScheduledNote } from '../core/music/Scheduler';
|
||||
import { dbToGain, SOUND_ITEM_MAX_DB } from '../core/music/song';
|
||||
import { DEFAULT_TEMPO } from '../core/music/time';
|
||||
import { type CutRange, cutChannels, cutRange, parseCutName } from '../core/model/sound';
|
||||
import type { AudioSink, ClipTiming, NoteTiming } from '../runtime/systems/Audio';
|
||||
import type { PitchOptions } from './pitchWorklet';
|
||||
import pitchWorklet from './pitchWorklet.ts?worker&url';
|
||||
@@ -39,6 +40,8 @@ const DECODE_RATE = 48000;
|
||||
const FADE = 0.01;
|
||||
/** How finely `peaks` cuts a sound up. */
|
||||
const PEAK_BUCKETS = 1024;
|
||||
/** Samples the cuts made so far may hold together before the oldest are let go; one is always kept. */
|
||||
const CUT_BUDGET = 64 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* One number of a voice that can still change while it plays (#315): a level, a share of a Mix, the
|
||||
@@ -221,6 +224,12 @@ export class SoundLibrary implements AudioSink {
|
||||
private context: AudioContext | null = null;
|
||||
private offline: OfflineAudioContext | null = null;
|
||||
private readonly buffers = new Map<string, AudioBuffer>();
|
||||
/**
|
||||
* The stretches of files that sound assets are cut to (#343), made from the whole file on first
|
||||
* play, oldest first. Unlike a file they can be made again at any time, so they are let go past a
|
||||
* budget: a cut dragged about in its tab would otherwise leave a copy of the sound per try.
|
||||
*/
|
||||
private readonly cuts = new Map<string, AudioBuffer>();
|
||||
/** The pitch worklet being loaded, and whether it is there once it is known; null before that. */
|
||||
private pitching: Promise<boolean> | null = null;
|
||||
private canPitch: boolean | null = null;
|
||||
@@ -286,7 +295,7 @@ export class SoundLibrary implements AudioSink {
|
||||
* same frame it asked for it, and the voice then starts with those values.
|
||||
*/
|
||||
play(file: string, volume: number, loop: boolean, owner = '', fx: FxVoice = DRY_VOICE): number {
|
||||
if (file === '' || this.failed.has(file)) return 0;
|
||||
if (file === '' || this.hasFailed(file)) return 0;
|
||||
const layered = { layers: [fx.plan], params: fx.params, tempo: fx.tempo ?? DEFAULT_TEMPO };
|
||||
return this.begin({ id: 0, file, files: [file], volume, loop, owner, fx: layered, duration: 0, at: 0 });
|
||||
}
|
||||
@@ -307,7 +316,7 @@ export class SoundLibrary implements AudioSink {
|
||||
timing: NoteTiming = {},
|
||||
): number {
|
||||
const files = [...new Set(layers.flatMap(filesOf))].filter(
|
||||
(file) => file !== '' && !this.failed.has(file),
|
||||
(file) => file !== '' && !this.hasFailed(file),
|
||||
);
|
||||
if (files.length === 0) return 0;
|
||||
const fx = { layers, params, tempo: timing.tempo ?? DEFAULT_TEMPO };
|
||||
@@ -322,7 +331,7 @@ export class SoundLibrary implements AudioSink {
|
||||
* gets its bytes late starts as far into the sound as it would be by then.
|
||||
*/
|
||||
playClip(file: string, clip: ClipTiming, fx: FxVoice = DRY_VOICE): number {
|
||||
if (file === '' || this.failed.has(file)) return 0;
|
||||
if (file === '' || this.hasFailed(file)) return 0;
|
||||
const layered = { layers: [fx.plan], params: fx.params, tempo: fx.tempo ?? DEFAULT_TEMPO };
|
||||
const { at, offset, duration, volume, bus } = clip;
|
||||
return this.begin({
|
||||
@@ -537,7 +546,7 @@ export class SoundLibrary implements AudioSink {
|
||||
private begin(start: Start): number {
|
||||
const id = this.nextId++;
|
||||
const asked: Start = { ...start, id };
|
||||
const loaded = start.files.every((file) => this.buffers.has(file));
|
||||
const loaded = start.files.every((file) => this.buffer(file) !== null);
|
||||
const pitching = start.fx.layers.some(pitches) ? this.pitchWorklet() : null;
|
||||
if (loaded && (!pitching || this.canPitch !== null)) {
|
||||
this.start(asked);
|
||||
@@ -598,6 +607,11 @@ export class SoundLibrary implements AudioSink {
|
||||
return voice ? this.tailOf(voice.fx) : 0;
|
||||
}
|
||||
|
||||
/** The second of the audio clock a voice started at; null while it waits on its bytes or is gone. */
|
||||
startedAt(id: number): number | null {
|
||||
return this.voices.find((voice) => voice.id === id)?.at ?? null;
|
||||
}
|
||||
|
||||
/** Whether a voice is still heard – or waiting on its bytes to be. A pattern or song counts as one. */
|
||||
playing(id: number): boolean {
|
||||
return this.sounding(id) || this.sequences.has(id);
|
||||
@@ -743,9 +757,23 @@ export class SoundLibrary implements AudioSink {
|
||||
return pitchWorklet;
|
||||
}
|
||||
|
||||
/** How long a decoded sound runs, in seconds; null while it has not been loaded (test hook). */
|
||||
/**
|
||||
* How long a decoded sound runs, in seconds – a cut (#343) as long as what it keeps; null while it
|
||||
* has not been loaded (test hook).
|
||||
*/
|
||||
duration(file: string): number | null {
|
||||
return this.buffers.get(file)?.duration ?? null;
|
||||
const range = this.rangeOf(file);
|
||||
return range ? (range.to - range.from) / range.whole.sampleRate : null;
|
||||
}
|
||||
|
||||
/** The decoded samples of a file as it is, without a cut; null while it has not been loaded. */
|
||||
decoded(file: string): AudioBuffer | null {
|
||||
return this.buffers.get(file) ?? null;
|
||||
}
|
||||
|
||||
/** Whether a file could not be fetched or decoded – it is not tried again. */
|
||||
failedToLoad(file: string): boolean {
|
||||
return this.hasFailed(file);
|
||||
}
|
||||
|
||||
length(file: string): number {
|
||||
@@ -759,13 +787,14 @@ export class SoundLibrary implements AudioSink {
|
||||
peaks(file: string): Float32Array | null {
|
||||
const known = this.peaked.get(file);
|
||||
if (known) return known;
|
||||
const buffer = this.buffers.get(file);
|
||||
if (!buffer) return null;
|
||||
const range = this.rangeOf(file);
|
||||
if (!range) return null;
|
||||
const { whole, from, to } = range;
|
||||
const peaks = new Float32Array(PEAK_BUCKETS);
|
||||
for (let c = 0; c < buffer.numberOfChannels; c++) {
|
||||
const data = buffer.getChannelData(c);
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
const bucket = Math.floor((i / data.length) * PEAK_BUCKETS);
|
||||
for (let c = 0; c < whole.numberOfChannels; c++) {
|
||||
const data = whole.getChannelData(c);
|
||||
for (let i = from; i < to; i++) {
|
||||
const bucket = Math.floor(((i - from) / (to - from)) * PEAK_BUCKETS);
|
||||
peaks[bucket] = Math.max(peaks[bucket]!, Math.abs(data[i]!));
|
||||
}
|
||||
}
|
||||
@@ -782,8 +811,15 @@ export class SoundLibrary implements AudioSink {
|
||||
for (const file of files) if (file) void this.load(file);
|
||||
}
|
||||
|
||||
/** Fetches and decodes once per file; a second play while it is in flight waits on the first. */
|
||||
/**
|
||||
* Fetches and decodes once per file; a second play while it is in flight waits on the first. A cut
|
||||
* (#343) loads its file and is made from it.
|
||||
*/
|
||||
async load(file: string): Promise<AudioBuffer | null> {
|
||||
if (parseCutName(file).cut) {
|
||||
const whole = await this.load(parseCutName(file).file);
|
||||
return whole ? this.buffer(file) : null;
|
||||
}
|
||||
const held = this.buffers.get(file);
|
||||
if (held) return held;
|
||||
if (this.failed.has(file)) return null;
|
||||
@@ -934,7 +970,13 @@ export class SoundLibrary implements AudioSink {
|
||||
const wiring: Wiring = {
|
||||
context,
|
||||
at: when,
|
||||
buffers: this.buffers,
|
||||
// Held here, not looked up later: a cut can be let go while its voice plays.
|
||||
buffers: new Map(
|
||||
files.flatMap((name) => {
|
||||
const buffer = this.buffer(name);
|
||||
return buffer ? [[name, buffer] as const] : [];
|
||||
}),
|
||||
),
|
||||
file,
|
||||
loop,
|
||||
rings,
|
||||
@@ -957,7 +999,7 @@ export class SoundLibrary implements AudioSink {
|
||||
layers: fx.layers.length,
|
||||
pitches: wiring.pitches,
|
||||
stages: wiring.stages,
|
||||
duration: Math.max(0, ...files.map((name) => this.buffers.get(name)?.duration ?? 0)),
|
||||
duration: Math.max(0, ...files.map((name) => this.duration(name) ?? 0)),
|
||||
at: when,
|
||||
envelopes: wiring.envelopes.size,
|
||||
};
|
||||
@@ -1173,6 +1215,50 @@ export class SoundLibrary implements AudioSink {
|
||||
return (this.context ??= new AudioContext());
|
||||
}
|
||||
|
||||
/** What a name plays, a cut made from its file the first time; null while the file is not loaded. */
|
||||
private buffer(name: string): AudioBuffer | null {
|
||||
const whole = this.buffers.get(name);
|
||||
if (whole) return whole;
|
||||
const held = this.cuts.get(name);
|
||||
if (held) {
|
||||
this.cuts.delete(name);
|
||||
this.cuts.set(name, held);
|
||||
return held;
|
||||
}
|
||||
const range = this.rangeOf(name);
|
||||
if (!range) return null;
|
||||
const made = this.decoder().createBuffer(
|
||||
range.whole.numberOfChannels,
|
||||
range.to - range.from,
|
||||
range.whole.sampleRate,
|
||||
);
|
||||
const channels = Array.from({ length: made.numberOfChannels }, (_, c) => range.whole.getChannelData(c));
|
||||
cutChannels(channels, range).forEach((data, c) => made.copyToChannel(data, c));
|
||||
this.cuts.set(name, made);
|
||||
const samples = (buffer: AudioBuffer) => buffer.length * buffer.numberOfChannels;
|
||||
let total = [...this.cuts.values()].reduce((sum, buffer) => sum + samples(buffer), 0);
|
||||
for (const [other, buffer] of this.cuts) {
|
||||
if (total <= CUT_BUDGET || other === name) break;
|
||||
this.cuts.delete(other);
|
||||
total -= samples(buffer);
|
||||
}
|
||||
return made;
|
||||
}
|
||||
|
||||
/** The samples of its loaded file a name plays: all of them, or the stretch a cut keeps. */
|
||||
private rangeOf(name: string): (CutRange & { whole: AudioBuffer }) | null {
|
||||
const { file, cut } = parseCutName(name);
|
||||
const whole = this.buffers.get(file);
|
||||
if (!whole) return null;
|
||||
if (!cut) return { whole, from: 0, to: whole.length, fadeIn: 0, fadeOut: 0 };
|
||||
return { whole, ...cutRange(whole.length, whole.sampleRate, cut) };
|
||||
}
|
||||
|
||||
/** Whether the file of a name could not be had: a cut of it cannot either. */
|
||||
private hasFailed(name: string): boolean {
|
||||
return this.failed.has(parseCutName(name).file);
|
||||
}
|
||||
|
||||
/** Loads the pitch worklet into the context once; says whether it is there. */
|
||||
private pitchWorklet(): Promise<boolean> {
|
||||
return (this.pitching ??= (async () => {
|
||||
|
||||
Reference in new issue
Block a user