From f1f4a57589c9cbede1e78c08c3613f3cd2532ad4 Mon Sep 17 00:00:00 2001 From: MelekTaus Date: Tue, 6 Oct 2026 21:42:14 +0200 Subject: [PATCH] Cut a sound to a stretch of its file in its own Sound tab (#343) A sound asset now opens in a tab that shows its file as a waveform. Two handles set where the sound starts and ends in the file, and that stretch is the sample: Play sound, a song item and an instrument's Sample all play it and nothing else. The file itself is never changed. - The asset has three new rows: Start (s), End (s) and Edge fade (ms). End 0 is the end of the file, the fade is 10 ms by default. Older assets have no values and play the whole file as before. - Every way of playing asks playedSoundOf() for a name instead of the file: the file, or #~~ for a cut. - SoundLibrary makes the cut from the decoded file the first time that name plays and ramps each edge inside the file over the fade. Length and peaks of a cut are read from the file without a copy. - Cut buffers can be made again at any time, so the library lets the oldest go past 64 Mi samples. A voice holds its buffers itself. - The tab draws min/max per pixel from 256-sample block peaks, and the samples as a line when zoomed in far. Wheel zooms, the middle button pans, as in the music tabs. Wave, cut and ruler sit under a mask, so a handle panned out of view does not draw over the panels beside. - Dragging a handle moves that edge, dragging elsewhere spans a new stretch. One drag is one undo step for both edges. Whole file puts the handles back, Play or Space plays the stretch with a playhead. - The inspector preview and the Sound FX preview play the cut too, and the song lanes draw and repeat it. Tests: - Unit: sounds 20, wave-view 8 (new), sound-library 68, plus 258 in the touched areas (instrument, song, pattern, inspector, tabs) - e2e 34 on melektaus-atx: ALL PASSED, 57 checks (7 new) - Gate on melektaus-atx: CHECK EXIT 0, 253 files, 3936 tests Co-Authored-By: Claude Opus 5.5 --- src/core/instrument/compile.ts | 8 +- src/core/model/sound.ts | 61 ++++ src/core/music/needs.ts | 3 +- src/core/properties/PropertySchema.ts | 26 +- src/core/properties/playedSound.ts | 19 + src/core/properties/sounds.ts | 17 + src/editor/Editor.ts | 11 +- src/editor/assetTabs.ts | 1 + src/editor/debug/soundHooks.ts | 44 +++ src/editor/debugHook.ts | 2 + src/editor/inspector/assetSection.ts | 4 +- src/editor/song/SongTimeline.ts | 24 +- src/editor/sound/SoundEditor.ts | 149 ++++++++ src/editor/sound/SoundPreview.ts | 37 +- src/editor/sound/WaveCanvas.ts | 458 +++++++++++++++++++++++++ src/editor/sound/waveView.ts | 188 ++++++++++ src/editor/soundfx/SoundFxEditor.ts | 3 +- src/editor/state.ts | 7 + src/runtime/systems/behaviorContext.ts | 10 +- src/viewport/SoundLibrary.ts | 116 ++++++- tests/core/sounds.test.ts | 86 ++++- tests/editor/wave-view.test.ts | 101 ++++++ tests/viewport/sound-library.test.ts | 18 + tools/e2e/34-sound.mjs | 102 ++++++ 24 files changed, 1442 insertions(+), 53 deletions(-) create mode 100644 src/core/properties/playedSound.ts create mode 100644 src/editor/debug/soundHooks.ts create mode 100644 src/editor/sound/SoundEditor.ts create mode 100644 src/editor/sound/WaveCanvas.ts create mode 100644 src/editor/sound/waveView.ts create mode 100644 tests/editor/wave-view.test.ts diff --git a/src/core/instrument/compile.ts b/src/core/instrument/compile.ts index 373ffb9b..209faaa6 100644 --- a/src/core/instrument/compile.ts +++ b/src/core/instrument/compile.ts @@ -15,6 +15,7 @@ import type { InstrumentAsset } from '../model/Asset'; import { graphLookup } from '../model/graphLookup'; import { textOf } from '../model/Properties'; import { findAssetOf, type Project } from '../model/Project'; +import { playedSoundOf } from '../properties/playedSound'; import { resolveAssetProps } from '../properties/resolve'; import { volumeOf } from '../properties/sounds'; import { soundFxBuildFor } from '../soundfx/compile'; @@ -71,7 +72,10 @@ export function instrumentSourcesOf(project: Project): InstrumentSources { lookup: graphLookup(project), sound: (assetId) => { const asset = findAssetOf(project, assetId, 'sound'); - return asset ? { file: asset.file, volume: volumeOf(resolveAssetProps(project, asset)) } : null; + if (!asset) return null; + // An asset without a file stays without one, so the Sample can say so. + const file = asset.file ? playedSoundOf(project, asset) : ''; + return { file, volume: volumeOf(resolveAssetProps(project, asset)) }; }, soundFx: (assetId) => { const asset = findAssetOf(project, assetId, 'soundfx'); @@ -79,7 +83,7 @@ export function instrumentSourcesOf(project: Project): InstrumentSources { }, key: (assetId) => { const sound = findAssetOf(project, assetId, 'sound'); - if (sound) return `${sound.file}@${volumeOf(resolveAssetProps(project, sound))}`; + if (sound) return `${playedSoundOf(project, sound)}@${volumeOf(resolveAssetProps(project, sound))}`; const fx = findAssetOf(project, assetId, 'soundfx'); return fx ? graphFingerprint(fx.graph) : ''; }, diff --git a/src/core/model/sound.ts b/src/core/model/sound.ts index e5d90baf..be733908 100644 --- a/src/core/model/sound.ts +++ b/src/core/model/sound.ts @@ -13,3 +13,64 @@ export function soundFileOf(project: Project, id: string): string { const asset = findAsset(project, id); return asset?.type === 'sound' ? asset.file : ''; } + +/** + * The stretch of its file a sound asset is (#343), in seconds: `end` 0 runs to the end of the file. + * `fade` is the ramp each edge inside the file gets, so a cut through a wave does not click. + */ +export interface SoundCut { + start: number; + end: number; + fade: number; +} + +/** Samples of a file a cut keeps, `from` up to `to`, and the samples of each ramp at an inner edge. */ +export interface CutRange { + from: number; + to: number; + fadeIn: number; + fadeOut: number; +} + +/** What a sound is played by: its file, or for a cut `#~~`, a name of its own. */ +export function cutName(file: string, cut: SoundCut): string { + if (file === '' || (cut.start <= 0 && cut.end <= 0)) return file; + return `${file}#${cut.start}~${cut.end}~${cut.fade}`; +} + +/** A name `cutName` made, taken apart again; a plain file has no cut. */ +export function parseCutName(name: string): { file: string; cut: SoundCut | null } { + const hash = name.indexOf('#'); + if (hash < 0) return { file: name, cut: null }; + const [start, end, fade] = name + .slice(hash + 1) + .split('~') + .map((part) => Number(part)); + const held = (value: number | undefined) => (Number.isFinite(value) ? Math.max(0, value!) : 0); + return { file: name.slice(0, hash), cut: { start: held(start), end: held(end), fade: held(fade) } }; +} + +/** + * Where a cut lies in `length` samples at `rate`: at least one sample, inside the file. An edge at + * the start or the end of the file is left as it is; one inside ramps over `fade`, no more than half + * of what is kept so the two ramps never cross. + */ +export function cutRange(length: number, rate: number, cut: SoundCut): CutRange { + const last = Math.max(1, length); + const from = Math.min(last - 1, Math.max(0, Math.round(cut.start * rate))); + const asked = cut.end > 0 ? Math.min(last, Math.round(cut.end * rate)) : last; + const to = Math.max(from + 1, asked); + const ramp = Math.min(Math.round(Math.max(0, cut.fade) * rate), Math.floor((to - from) / 2)); + return { from, to, fadeIn: from > 0 ? ramp : 0, fadeOut: to < last ? ramp : 0 }; +} + +/** The samples of each channel a cut keeps, its inner edges ramped in and out. */ +export function cutChannels(channels: readonly Float32Array[], range: CutRange): Float32Array[] { + const { from, to, fadeIn, fadeOut } = range; + return channels.map((data) => { + const kept = data.slice(from, to); + for (let i = 0; i < fadeIn; i++) kept[i]! *= i / fadeIn; + for (let i = 0; i < fadeOut; i++) kept[kept.length - 1 - i]! *= i / fadeOut; + return kept; + }); +} diff --git a/src/core/music/needs.ts b/src/core/music/needs.ts index af444d3f..5d786f06 100644 --- a/src/core/music/needs.ts +++ b/src/core/music/needs.ts @@ -1,5 +1,6 @@ import { instrumentBuildFor, instrumentSourcesOf } from '../instrument/compile'; import { findAssetOf, type Project } from '../model/Project'; +import { playedSoundById } from '../properties/playedSound'; import { soundFxBuildFor } from '../soundfx/compile'; import { DRY_VOICE, type FxPlan, type FxVoice } from '../soundfx/nodes'; import type { SequenceNeeds } from './playbacks'; @@ -26,7 +27,7 @@ export function sequenceNeeds(project: Project, sequence: Sequence): SequenceNee } const files = new Set(); for (const clip of sequence.clips ?? []) { - const file = findAssetOf(project, clip.sound, 'sound')?.file; + const file = playedSoundById(project, clip.sound); if (file) files.add(file); const fx = clipFx(project, clip.fx, 120); if (fx !== DRY_VOICE) layers.push(fx.plan); diff --git a/src/core/properties/PropertySchema.ts b/src/core/properties/PropertySchema.ts index ea199305..c123a593 100644 --- a/src/core/properties/PropertySchema.ts +++ b/src/core/properties/PropertySchema.ts @@ -103,7 +103,14 @@ import { } from './particles'; import { SHADER_KEY, shaderOf, shaderParamDefs } from './shaders'; import { SHADER_RESOLUTION_KEY, SHADER_RESOLUTION_OPTIONS } from './cameraShader'; -import { LOOP_KEY as SOUND_LOOP_KEY, VOLUME_KEY as SOUND_VOLUME_KEY } from './sounds'; +import { + DEFAULT_FADE_MS, + END_KEY as SOUND_END_KEY, + FADE_KEY as SOUND_FADE_KEY, + LOOP_KEY as SOUND_LOOP_KEY, + START_KEY as SOUND_START_KEY, + VOLUME_KEY as SOUND_VOLUME_KEY, +} from './sounds'; import { BEAT_OPTIONS, DEFAULT_BARS, @@ -671,11 +678,24 @@ const ASSET_SCHEMAS: Record = { // The typeface itself is intrinsic data on the asset (see FontAsset), not a property. Nearest draws // a pixel font a pixel per unit, magnified sharp. font: [filterRow(true)], - // The file is intrinsic data (see SoundAsset); these two say how it is played by default, and a - // `Play sound` node scales the one and can add to the other. + // The file is intrinsic data (see SoundAsset); volume and loop say how it is played by default, and + // a `Play sound` node scales the one and can add to the other. sound: [ { key: SOUND_VOLUME_KEY, label: 'Volume', kind: 'number', default: 1, min: 0, max: 1, step: 0.05 }, { key: SOUND_LOOP_KEY, label: 'Loop', kind: 'boolean', default: false }, + // The stretch of the file it is (#343), set in its tab; an end of 0 is the end of the file. + { key: SOUND_START_KEY, label: 'Start (s)', kind: 'number', default: 0, min: 0, step: 0.01, decimals: 3 }, + { key: SOUND_END_KEY, label: 'End (s)', kind: 'number', default: 0, min: 0, step: 0.01, decimals: 3 }, + { + key: SOUND_FADE_KEY, + label: 'Edge fade (ms)', + kind: 'number', + default: DEFAULT_FADE_MS, + min: 0, + max: 1000, + step: 1, + decimals: 0, + }, ], // The effects are the asset's graph (see SoundFxAsset), not a property. soundfx: [], diff --git a/src/core/properties/playedSound.ts b/src/core/properties/playedSound.ts new file mode 100644 index 00000000..85417038 --- /dev/null +++ b/src/core/properties/playedSound.ts @@ -0,0 +1,19 @@ +import type { SoundAsset } from '../model/Asset'; +import { findAssetOf, type Project } from '../model/Project'; +import { cutName } from '../model/sound'; +import { resolveAssetProps } from './resolve'; +import { cutOf } from './sounds'; + +/** + * What a sound asset is played by (#343): its file, or the stretch of it the asset is cut to. Every + * way a sound is heard asks this rather than for the file – the cut is the sample. + */ +export function playedSoundOf(project: Project, asset: SoundAsset): string { + return cutName(asset.file, cutOf(resolveAssetProps(project, asset))); +} + +/** The same by id, empty when the id is unset or no longer names a sound. */ +export function playedSoundById(project: Project, id: string): string { + const asset = id === '' ? undefined : findAssetOf(project, id, 'sound'); + return asset ? playedSoundOf(project, asset) : ''; +} diff --git a/src/core/properties/sounds.ts b/src/core/properties/sounds.ts index 28d2910c..c908da15 100644 --- a/src/core/properties/sounds.ts +++ b/src/core/properties/sounds.ts @@ -1,10 +1,17 @@ import { booleanOf, type PropertyBag } from '../model/Properties'; +import type { SoundCut } from '../model/sound'; import { coerceAs } from './kinds'; /** How loud this sound is by itself, 0–1; a `Play sound` node scales this rather than replacing it. */ export const VOLUME_KEY = 'volume'; /** Whether this sound is music: it repeats until something stops it. */ export const LOOP_KEY = 'loop'; +/** Where in its file the sound starts and ends, in seconds; an end of 0 is the end of the file (#343). */ +export const START_KEY = 'start'; +export const END_KEY = 'end'; +/** Milliseconds each cut edge inside the file ramps over, so it does not click. */ +export const FADE_KEY = 'fade'; +export const DEFAULT_FADE_MS = 10; export function volumeOf(bag: PropertyBag): number { return coerceAs(bag[VOLUME_KEY], 'number', 1); @@ -13,3 +20,13 @@ export function volumeOf(bag: PropertyBag): number { export function loopsOf(bag: PropertyBag): boolean { return booleanOf(bag[LOOP_KEY]); } + +/** The stretch of its file a sound is, its fade in seconds. */ +export function cutOf(bag: PropertyBag): SoundCut { + const seconds = (key: string, fallback: number) => Math.max(0, coerceAs(bag[key], 'number', fallback)); + return { + start: seconds(START_KEY, 0), + end: seconds(END_KEY, 0), + fade: seconds(FADE_KEY, DEFAULT_FADE_MS) / 1000, + }; +} diff --git a/src/editor/Editor.ts b/src/editor/Editor.ts index 5957536f..0285d7c6 100644 --- a/src/editor/Editor.ts +++ b/src/editor/Editor.ts @@ -84,6 +84,7 @@ import { copySelection } from './clipboard'; import { PixelArtEditor } from './pixelart/PixelArtEditor'; import { VectorArtEditor } from './vectorart/VectorArtEditor'; import { CurveEditor } from './curve/CurveEditor'; +import { SoundEditor } from './sound/SoundEditor'; import { TextureEditor } from './texture/TextureEditor'; import { installFileImport } from './fileImport'; import type { PluginManager } from './plugins/loader'; @@ -97,6 +98,7 @@ export type CentreTab = | 'statemachine' | 'shader' | 'particles' + | 'sound' | 'soundfx' | 'instrument' | 'pattern' @@ -150,6 +152,7 @@ const PANE_ORDER = [ 'statemachine', 'shader', 'particles', + 'sound', 'soundfx', 'instrument', 'pattern', @@ -173,6 +176,7 @@ const TAB_LABELS: Record = { statemachine: 'State machine', shader: 'Shader', particles: 'Particles', + sound: 'Sound', soundfx: 'Sound FX', instrument: 'Instrument', pattern: 'Pattern', @@ -225,6 +229,7 @@ export class Editor implements PlayHost { readonly stateMachineEditor: StateMachineEditor; readonly shaderEditor: ShaderEditor; readonly particlesEditor: ParticlesEditor; + readonly soundEditor: SoundEditor; readonly soundFxEditor: SoundFxEditor; readonly instrumentEditor: InstrumentEditor; readonly patternEditor: PatternEditor; @@ -314,6 +319,7 @@ export class Editor implements PlayHost { this.stateMachineEditor = new StateMachineEditor(state); this.shaderEditor = new ShaderEditor(state); this.particlesEditor = new ParticlesEditor(state); + this.soundEditor = new SoundEditor(state); this.soundFxEditor = new SoundFxEditor(state); this.instrumentEditor = new InstrumentEditor(state); this.patternEditor = new PatternEditor(state); @@ -426,6 +432,7 @@ export class Editor implements PlayHost { statemachine: this.stateMachineEditor, shader: this.shaderEditor, particles: this.particlesEditor, + sound: this.soundEditor, soundfx: this.soundFxEditor, instrument: this.instrumentEditor, pattern: this.patternEditor, @@ -1023,6 +1030,7 @@ export class Editor implements PlayHost { if (this.tab === 'curve') this.curveEditor.tick(dt); if (this.tab === 'pattern') this.patternEditor.tick(); if (this.tab === 'song') this.songEditor.tick(); + if (this.tab === 'sound') this.soundEditor.tick(); // Play repaints the machine tab itself (see `PlaySession.tick`); an editor run has no session // to do it, and its states move just the same. if (this.tab === 'statemachine' && this.editorRun.isRunning) this.stateMachineEditor.canvas.redraw(); @@ -1057,7 +1065,8 @@ export class Editor implements PlayHost { (this.tab === 'tile' && this.tileEditor.isPlaying) || (this.tab === 'curve' && this.curveEditor.isPlaying) || (this.tab === 'pattern' && this.patternEditor.isPlaying) || - (this.tab === 'song' && this.songEditor.isPlaying) + (this.tab === 'song' && this.songEditor.isPlaying) || + (this.tab === 'sound' && this.soundEditor.isPlaying) ); } diff --git a/src/editor/assetTabs.ts b/src/editor/assetTabs.ts index 8cf96e58..b74f13da 100644 --- a/src/editor/assetTabs.ts +++ b/src/editor/assetTabs.ts @@ -9,6 +9,7 @@ export const TAB_OF_ASSET: Partial> = { shader: 'shader', shaderfunction: 'shader', particles: 'particles', + sound: 'sound', soundfx: 'soundfx', instrument: 'instrument', pattern: 'pattern', diff --git a/src/editor/debug/soundHooks.ts b/src/editor/debug/soundHooks.ts new file mode 100644 index 00000000..e6f68c7b --- /dev/null +++ b/src/editor/debug/soundHooks.ts @@ -0,0 +1,44 @@ +import { findAssetOf } from '../../core/model/Project'; +import { playedSoundOf } from '../../core/properties/playedSound'; +import { soundLibrary } from '../../viewport/SoundLibrary'; +import { centreOf, type HookContext } from './hookHelpers'; + +/** The Sound tab (#343). */ +export function soundHooks({ state, editor }: HookContext) { + const { soundEditor } = editor; + return { + soundTab: { + current: () => soundEditor.currentAssetId, + info: () => soundEditor.shownInfo, + rect: () => soundEditor.canvasRect(), + /** Where a second of the file is on screen, at the middle of the wave. */ + at: (seconds: number) => soundEditor.canvas.secondsToScreen(seconds), + /** The stretch between the handles, in seconds; null while the file is not there. */ + span: () => soundEditor.canvas.shownSpan(), + duration: () => soundEditor.canvas.fileDuration(), + cursor: () => soundEditor.canvas.cursor, + playhead: () => soundEditor.canvas.playhead, + playing: () => soundEditor.isPlaying, + playButton: () => ({ ...centreOf(soundEditor.playButton), text: soundEditor.playButton.text }), + wholeButton: () => ({ ...centreOf(soundEditor.wholeButton), enabled: soundEditor.wholeButton.enabled }), + fadeField: () => ({ ...centreOf(soundEditor.fadeField), text: soundEditor.fadeField.value }), + /** What a sound asset is played by: its file, or the name of its cut. */ + played: (assetId: string) => { + const asset = findAssetOf(state.project, assetId, 'sound'); + return asset ? playedSoundOf(state.project, asset) : null; + }, + /** The samples a name plays, cut and faded: their count, rate, and the first and last few of channel 0. */ + samples: async (name: string) => { + const buffer = await soundLibrary.load(name); + if (!buffer) return null; + const data = buffer.getChannelData(0); + return { + length: buffer.length, + rate: buffer.sampleRate, + head: Array.from(data.slice(0, 4)), + tail: Array.from(data.slice(-4)), + }; + }, + }, + }; +} diff --git a/src/editor/debugHook.ts b/src/editor/debugHook.ts index ce8db7fc..921b9407 100644 --- a/src/editor/debugHook.ts +++ b/src/editor/debugHook.ts @@ -13,6 +13,7 @@ import { stageNodes, } from '../viewport/SoundLibrary'; import { curveHooks } from './debug/curveHooks'; +import { soundHooks } from './debug/soundHooks'; import { patternHooks } from './debug/patternHooks'; import { songHooks } from './debug/songHooks'; import { documentHooks } from './debug/documentHooks'; @@ -48,6 +49,7 @@ export function installDebugHook(app: Application, state: EditorState, editor: E ...curveHooks(context), ...patternHooks(context), ...songHooks(context), + ...soundHooks(context), ...renderHooks(context), /** * Voices started, stopped and in flight. CI runs the browser with `--mute-audio`, so nothing diff --git a/src/editor/inspector/assetSection.ts b/src/editor/inspector/assetSection.ts index e8651871..982b604f 100644 --- a/src/editor/inspector/assetSection.ts +++ b/src/editor/inspector/assetSection.ts @@ -167,10 +167,10 @@ export function buildAssetSection(host: InspectorHost, assetId: string): void { host.body.add( asset.file ? new FontPreview(asset.file) : new Label('Not uploaded.', { style: 'warning' }), ); - // A sound has two rows above this; the preview is the only way to find out which sound it is. + // A sound has its rows above this; the preview is the quickest way to find out which sound it is. if (asset.type === 'sound') host.body.add( - asset.file ? new SoundPreview(asset.file) : new Label('Not uploaded.', { style: 'warning' }), + asset.file ? new SoundPreview(host.state, assetId) : new Label('Not uploaded.', { style: 'warning' }), ); if (asset.type === 'sprite') addSpritePreview(host, { kind: 'asset', assetId }); } diff --git a/src/editor/song/SongTimeline.ts b/src/editor/song/SongTimeline.ts index d256088e..74a06c14 100644 --- a/src/editor/song/SongTimeline.ts +++ b/src/editor/song/SongTimeline.ts @@ -42,6 +42,7 @@ import { Widget, zoomFactor, } from '../../ui'; +import { playedSoundById, playedSoundOf } from '../../core/properties/playedSound'; import { soundLibrary } from '../../viewport/SoundLibrary'; import { TimeRuler } from '../music/TimeRuler'; import { @@ -940,8 +941,9 @@ export class SongTimeline extends Widget { if (dragged.type === 'pattern') length = droppedLength(null, patternLength(dragged), songTempo(asset.props), bar); else { - this.ensureLoaded(dragged.file); - length = droppedLength(soundLibrary.duration(dragged.file), null, songTempo(asset.props), bar); + const played = playedSoundOf(this.state.project, dragged); + this.ensureLoaded(played); + length = droppedLength(soundLibrary.duration(played), null, songTempo(asset.props), bar); } this.drop = { kind: dragged.type, asset: dragged.id, track: asset.tracks[row]!.id, tick, length }; this.invalidate(); @@ -1092,8 +1094,8 @@ export class SongTimeline extends Widget { const pattern = findAssetOf(this.state.project, item.asset, 'pattern'); return pattern ? patternLength(pattern) : null; } - const sound = findAssetOf(this.state.project, item.asset, 'sound'); - const seconds = sound?.file ? soundLibrary.duration(sound.file) : null; + const played = playedSoundById(this.state.project, item.asset); + const seconds = played ? soundLibrary.duration(played) : null; return seconds ? ticksOfSeconds(seconds, tempo) : null; } @@ -1148,8 +1150,8 @@ export class SongTimeline extends Widget { const pattern = findAssetOf(this.state.project, item.asset, 'pattern'); return pattern && pattern.notes.length > 0 ? 'notes' : 'none'; } - const sound = findAssetOf(this.state.project, item.asset, 'sound'); - return sound?.file && soundLibrary.peaks(sound.file) ? 'wave' : 'none'; + const played = playedSoundById(this.state.project, item.asset); + return played && soundLibrary.peaks(played) ? 'wave' : 'none'; } private ensureLoaded(file: string): void { @@ -1397,12 +1399,12 @@ export class SongTimeline extends Widget { * round is drawn from its own start in the same columns, so every repeat looks alike. */ private drawWave(item: SongItem, box: Box, lw: number): void { - const sound = findAssetOf(this.state.project, item.asset, 'sound'); - if (!sound?.file) return; - const peaks = soundLibrary.peaks(sound.file); - const seconds = soundLibrary.duration(sound.file); + const played = playedSoundById(this.state.project, item.asset); + if (!played) return; + const peaks = soundLibrary.peaks(played); + const seconds = soundLibrary.duration(played); if (!peaks || !seconds) { - this.ensureLoaded(sound.file); + this.ensureLoaded(played); return; } const asset = this.asset()!; diff --git a/src/editor/sound/SoundEditor.ts b/src/editor/sound/SoundEditor.ts new file mode 100644 index 00000000..d85ce71a --- /dev/null +++ b/src/editor/sound/SoundEditor.ts @@ -0,0 +1,149 @@ +import { SetAssetPropertyCommand } from '../../core/commands/assetCommands'; +import type { Asset, SoundAsset } from '../../core/model/Asset'; +import { playedSoundOf } from '../../core/properties/playedSound'; +import { resolveAssetProps } from '../../core/properties/resolve'; +import { cutOf, FADE_KEY, volumeOf } from '../../core/properties/sounds'; +import { Button, Label, NumberField } from '../../ui'; +import { soundLibrary } from '../../viewport/SoundLibrary'; +import { AssetTabEditor } from '../AssetTabEditor'; +import { claimTransport, releaseTransport } from '../music/transport'; +import { TOOLBAR_ICON_SVG } from '../panels/toolbarIcons'; +import type { EditorState } from '../state'; +import { WaveCanvas } from './WaveCanvas'; + +const INFO_WIDTH = 140; +const FADE_WIDTH = 56; +const PREVIEW_OWNER = 'sound-preview'; + +/** + * The Sound tab (#343): the file as a waveform, and the stretch of it the asset is. That stretch is + * the sample – a song item, a `Play sound` and an instrument's Sample all play it and nothing else. + * The header plays it (Space does too), sets the fade at its edges and puts the whole file back. + */ +export class SoundEditor extends AssetTabEditor { + /** Public for the check scripts. */ + readonly playButton: Button; + readonly wholeButton: Button; + readonly fadeField: NumberField; + /** The voice the tab plays, 0 while none, and the sound it is of. */ + private voice = 0; + private playingId = ''; + private readonly halt = () => this.stop(); + + constructor(state: EditorState) { + super(state, { infoWidth: INFO_WIDTH }); + this.playButton = new Button('Play', { + icon: TOOLBAR_ICON_SVG.play, + minWidth: 72, + tooltip: 'Play the stretch between the handles (Space)', + onClick: () => this.togglePlay(), + }); + this.wholeButton = new Button('Whole file', { + tooltip: 'Play the whole file again: the handles go back to its start and end', + onClick: () => this.canvas.resetCut(), + }); + this.fadeField = new NumberField(0, { + width: FADE_WIDTH, + min: 0, + max: 1000, + step: 1, + decimals: 0, + onChange: (ms) => this.setFade(ms), + }); + const canvas = new WaveCanvas(state); + canvas.onTogglePlay = () => this.togglePlay(); + canvas.onLoaded = () => this.refresh(); + this.mount(canvas, [ + this.playButton, + this.wholeButton, + new Label('Edge fade (ms)', { style: 'muted' }), + this.fadeField, + ]); + state.on('load', () => this.stop()); + } + + override selectedId(): string { + return this.state.selectedSoundId ?? ''; + } + + override isMine(asset: Asset): asset is SoundAsset { + return asset.type === 'sound'; + } + + get isPlaying(): boolean { + return this.voice !== 0; + } + + togglePlay(): void { + if (this.isPlaying) this.stop(); + else this.play(); + } + + /** Plays the stretch once, at the sound's own volume, from its start. */ + play(): void { + const asset = this.asset(); + if (!asset?.file) return; + this.stop(); + claimTransport(this.halt); + const volume = volumeOf(resolveAssetProps(this.state.project, asset)); + this.voice = soundLibrary.play(playedSoundOf(this.state.project, asset), volume, false, PREVIEW_OWNER); + this.playingId = this.voice ? asset.id : ''; + this.syncPlayButton(); + } + + stop(): void { + releaseTransport(this.halt); + if (this.voice) soundLibrary.stopOwner(PREVIEW_OWNER); + this.voice = 0; + this.playingId = ''; + this.canvas.playhead = null; + this.canvas.invalidate(); + this.syncPlayButton(); + } + + /** Only called while the tab is visible: the playhead follows what is heard, and stops with it. */ + tick(): void { + if (!this.voice) return; + if (!soundLibrary.playing(this.voice)) return this.stop(); + const started = soundLibrary.startedAt(this.voice); + const span = this.canvas.shownSpan(); + if (started === null || !span) return; + const heard = soundLibrary.clock() - started - soundLibrary.latency(); + this.canvas.playhead = span.start + Math.max(0, Math.min(span.end - span.start, heard)); + this.canvas.invalidate(); + } + + private setFade(ms: number): void { + const asset = this.asset(); + const now = asset && Math.round(cutOf(resolveAssetProps(this.state.project, asset)).fade * 1000); + if (!asset || ms === now) return; + this.state.execute(new SetAssetPropertyCommand(asset.id, FADE_KEY, ms)); + } + + private syncPlayButton(): void { + this.playButton.setIcon(this.isPlaying ? TOOLBAR_ICON_SVG.stop : TOOLBAR_ICON_SVG.play); + this.playButton.setText(this.isPlaying ? 'Stop' : 'Play'); + this.playButton.setEnabled(Boolean(this.asset()?.file)); + } + + protected override infoText(asset: SoundAsset): string { + const span = this.canvas.shownSpan(); + const whole = this.canvas.fileDuration(); + if (!asset.file) return 'Not uploaded'; + if (!span || whole === null) return ''; + const kept = span.end - span.start; + return kept >= whole ? `${whole.toFixed(2)} s` : `${kept.toFixed(2)} s of ${whole.toFixed(2)} s`; + } + + protected override syncExtras(asset: SoundAsset | null): void { + const ready = asset !== null && Boolean(asset.file); + this.fitButton.setEnabled(ready); + this.wholeButton.setEnabled(ready); + this.fadeField.setEnabled(ready); + const fade = asset ? cutOf(resolveAssetProps(this.state.project, asset)).fade : 0; + this.fadeField.setNumber(Math.round(fade * 1000)); + // Another sound on screen: what was playing is no longer what the tab shows. + if (this.isPlaying && asset?.id !== this.playingId) this.stop(); + this.syncPlayButton(); + } +} diff --git a/src/editor/sound/SoundPreview.ts b/src/editor/sound/SoundPreview.ts index 82a9abe5..ab33d9f4 100644 --- a/src/editor/sound/SoundPreview.ts +++ b/src/editor/sound/SoundPreview.ts @@ -1,29 +1,40 @@ +import { playedSoundById } from '../../core/properties/playedSound'; import { Button, HStack, Label, Theme } from '../../ui'; import { soundLibrary } from '../../viewport/SoundLibrary'; import { TOOLBAR_ICON_SVG } from '../panels/toolbarIcons'; +import type { EditorState } from '../state'; /** - * What a sound asset looks like: a button that plays it and the length the decoder reports. There - * is nothing else to show – its file name is a hash, and the waveform would be a picture of bytes - * nobody reads. + * What a sound asset sounds like: a button that plays it and the length the decoder reports. Both + * are of the stretch the asset is cut to (#343); the waveform is in its tab. * * Playing here goes through the same library a running game uses, so what the inspector plays is * exactly what the graph will: same decode, same device, and `Stop` reaches both. */ export class SoundPreview extends HStack { private readonly length: Label; + private shown = ''; + private readonly stopWatching: () => void; - constructor(private readonly file: string) { + constructor( + private readonly state: EditorState, + private readonly assetId: string, + ) { super({ gap: Theme.gap, align: 'center' }); const play = new Button('Play', { icon: TOOLBAR_ICON_SVG.play }); - play.onClick = () => soundLibrary.play(this.file, 1, false); + play.onClick = () => soundLibrary.play(this.played(), 1, false); this.add(play); const stop = new Button('Stop', { icon: TOOLBAR_ICON_SVG.stop }); - stop.onClick = () => soundLibrary.stop(this.file); + stop.onClick = () => soundLibrary.stop(this.played()); this.add(stop); this.length = new Label('', { style: 'muted' }); this.add(this.length, 1); this.showLength(); + this.stopWatching = state.on('project', () => this.showLength()); + } + + private played(): string { + return playedSoundById(this.state.project, this.assetId); } /** @@ -32,14 +43,22 @@ export class SoundPreview extends HStack { * which is more honest than a number guessed from the file size. */ private showLength(): void { - const known = soundLibrary.duration(this.file); + const name = this.played(); + if (name === this.shown) return; + this.shown = name; + const known = soundLibrary.duration(name); if (known !== null) { this.length.setText(`${known.toFixed(2)} s`); return; } - void soundLibrary.load(this.file).then((buffer) => { - if (this.destroyed) return; + void soundLibrary.load(name).then((buffer) => { + if (this.destroyed || this.shown !== name) return; this.length.setText(buffer ? `${buffer.duration.toFixed(2)} s` : 'Could not be decoded.'); }); } + + override destroy(options?: Parameters[0]): void { + this.stopWatching(); + super.destroy(options); + } } diff --git a/src/editor/sound/WaveCanvas.ts b/src/editor/sound/WaveCanvas.ts new file mode 100644 index 00000000..b2aeec2c --- /dev/null +++ b/src/editor/sound/WaveCanvas.ts @@ -0,0 +1,458 @@ +import { + Container, + type FederatedPointerEvent, + type FederatedWheelEvent, + Graphics, + type Text, +} from 'pixi.js'; +import { CompoundCommand } from '../../core/commands/CompoundCommand'; +import { SetAssetPropertyCommand } from '../../core/commands/assetCommands'; +import type { SoundAsset } from '../../core/model/Asset'; +import type { Vec2 } from '../../core/model/Properties'; +import { findAssetOf } from '../../core/model/Project'; +import { cutRange } from '../../core/model/sound'; +import { resolveAssetProps } from '../../core/properties/resolve'; +import { cutOf, END_KEY, START_KEY } from '../../core/properties/sounds'; +import { crispText, type Size, textStyle, Theme, ui, Widget, zoomFactor } from '../../ui'; +import { soundLibrary } from '../../viewport/SoundLibrary'; +import type { EditorState } from '../state'; +import { + clampWave, + type CutSpan, + draggedSpan, + edgeAt, + fitWave, + rulerLabel, + rulerStep, + secondsToX, + spanOf, + spanPeak, + storedCut, + type WavePeaks, + wavePeaks, + type WaveView, + xToSeconds, + zoomWave, +} from './waveView'; + +const RULER_HEIGHT = 24; +/** Room either side of the file when it is fitted. */ +const FIT_MARGIN = 16; +/** Pixels apart the ruler's labels are at least. */ +const LABEL_SPACING = 70; +/** The shortest span a drag leaves. */ +const LEAST_SPAN = 0.001; +/** Pixels a press has to travel before it spans a new stretch. */ +const DRAG_START = 3; + +type Gesture = + | { kind: 'pan'; pointerX: number; panX: number } + | { + kind: 'span'; + edge: 'start' | 'end' | null; + from: number; + pointerX: number; + moved: boolean; + span: CutSpan; + }; + +/** A file's samples as the canvas draws them: decoded once, its block peaks made on first sight. */ +interface Wave { + duration: number; + rate: number; + channels: Float32Array[]; + peaks: WavePeaks; +} + +const waves = new WeakMap(); + +function waveOf(buffer: AudioBuffer): Wave { + const known = waves.get(buffer); + if (known) return known; + const channels = Array.from({ length: buffer.numberOfChannels }, (_, c) => buffer.getChannelData(c)); + const wave = { duration: buffer.duration, rate: buffer.sampleRate, channels, peaks: wavePeaks(channels) }; + waves.set(buffer, wave); + return wave; +} + +/** + * The Sound tab's canvas (#343): the whole file as one waveform, the loudest of its channels, under + * a ruler in seconds. The stretch the asset is cut to stands between two handles, what lies outside + * it darker; dragging a handle moves that edge, dragging beside them spans a new stretch. One drag + * is one undo step. The wheel zooms, the middle button pans, as in the music tabs. + */ +export class WaveCanvas extends Widget { + private readonly bg = new Graphics(); + private readonly waveG = new Graphics(); + private readonly overlay = new Graphics(); + private readonly labels = new Container(); + /** Wave, cut and ruler, kept to the widget: a handle panned out of view must not draw beside it. */ + private readonly content = new Container(); + private readonly clip = new Graphics(); + private readonly texts: Text[] = []; + private readonly hint = crispText({ text: '', style: textStyle('muted') }); + private view: WaveView = { panX: FIT_MARGIN, pxPerSecond: 100 }; + private needsFit = true; + private assetId = ''; + private gesture: Gesture | null = null; + /** The files asked for, so a wave is drawn once its bytes are there. */ + private readonly loading = new Set(); + /** Where the tab's playing stands in the file, in seconds; null while it does not play. */ + playhead: number | null = null; + onTogglePlay: () => void = () => {}; + /** The file's samples arrived: the tab's readout can say how long it is. */ + onLoaded: () => void = () => {}; + + constructor(private readonly state: EditorState) { + super(); + this.focusable = true; + this.labels.eventMode = 'none'; + this.hint.eventMode = 'none'; + this.content.addChild(this.waveG, this.overlay, this.labels); + this.content.mask = this.clip; + this.addChild(this.bg, this.content, this.clip, this.hint); + this.on('pointerdown', (e: FederatedPointerEvent) => this.onPointerDown(e)); + this.on('globalpointermove', (e: FederatedPointerEvent) => this.onPointerMove(e)); + this.onPointerRelease(() => this.onPointerUp()); + this.on('wheel', (e: FederatedWheelEvent) => this.onWheel(e)); + state.on('project', () => this.invalidate()); + } + + override measure(): Size { + return { w: Math.max(this.minWidth, 360), h: Math.max(this.minHeight, 160) }; + } + + get currentAssetId(): string { + return this.assetId; + } + + show(assetId: string): void { + if (this.assetId === assetId) return; + this.gesture = null; + this.assetId = assetId; + this.needsFit = true; + this.redraw(); + } + + saveView(): WaveView { + return { ...this.view }; + } + + restoreView(view: unknown): void { + const held = view as Partial; + if (typeof held.panX !== 'number' || typeof held.pxPerSecond !== 'number') return; + this.view = { panX: held.panX, pxPerSecond: held.pxPerSecond }; + this.needsFit = false; + this.redraw(); + } + + /** The whole file across. */ + fit(): void { + const wave = this.wave(); + if (!wave || this.w <= 0) { + this.needsFit = true; + return; + } + this.view = fitWave(wave.duration, this.w, FIT_MARGIN); + this.needsFit = false; + this.redraw(); + } + + /** The stretch the asset is cut to, as shown – while a drag runs, as it would leave it. */ + shownSpan(): CutSpan | null { + const asset = this.asset(); + const wave = this.wave(); + if (!asset || !wave) return null; + if (this.gesture?.kind === 'span') return this.gesture.span; + const cut = cutOf(resolveAssetProps(this.state.project, asset)); + return spanOf(cut.start, cut.end, wave.duration); + } + + /** The file's length in seconds, null while it is not loaded (test hook). */ + fileDuration(): number | null { + return this.wave()?.duration ?? null; + } + + /** Where a second of the file is on screen, at the middle of the wave (test hook). */ + secondsToScreen(seconds: number): Vec2 { + const origin = this.getGlobalPosition(); + return { x: origin.x + secondsToX(this.view, seconds), y: origin.y + RULER_HEIGHT + this.waveHeight / 2 }; + } + + /** Puts the whole file back, in one undo step. */ + resetCut(): void { + const asset = this.asset(); + const wave = this.wave(); + if (asset && wave) this.commit(asset, { start: 0, end: wave.duration }, wave, 'Reset sound cut'); + } + + private asset(): SoundAsset | null { + return (this.assetId && findAssetOf(this.state.project, this.assetId, 'sound')) || null; + } + + /** The file's samples, asked for the first time they are not there. */ + private wave(): Wave | null { + const file = this.asset()?.file; + if (!file) return null; + const buffer = soundLibrary.decoded(file); + if (buffer) return waveOf(buffer); + if (!this.loading.has(file)) { + this.loading.add(file); + void soundLibrary.load(file).then(() => { + this.invalidate(); + this.onLoaded(); + }); + } + return null; + } + + private get waveHeight(): number { + return Math.max(0, this.h - RULER_HEIGHT); + } + + override onKeyDown(event: KeyboardEvent): boolean { + if (event.code !== 'Space' || event.ctrlKey || event.metaKey || !this.asset()) return false; + this.onTogglePlay(); + return true; + } + + private onPointerDown(e: FederatedPointerEvent): void { + const wave = this.wave(); + const span = this.shownSpan(); + if (!wave || !span) return; + const p = this.toLocal(e.global); + e.stopPropagation(); + ui().focus.focus(this); + if (e.button === 1) { + this.gesture = { kind: 'pan', pointerX: p.x, panX: this.view.panX }; + return; + } + if (e.button !== 0) return; + const from = Math.max(0, Math.min(wave.duration, xToSeconds(this.view, p.x))); + this.gesture = { + kind: 'span', + edge: edgeAt(this.view, span, p.x), + from, + pointerX: p.x, + moved: false, + span, + }; + } + + private onPointerMove(e: FederatedPointerEvent): void { + const p = this.toLocal(e.global); + const g = this.gesture; + const wave = this.wave(); + if (!g || !wave) { + this.updateCursor(p.x, p.y); + return; + } + if (g.kind === 'pan') { + this.view = clampWave({ ...this.view, panX: g.panX + p.x - g.pointerX }, wave.duration, this.w); + this.needsFit = false; + this.redraw(); + return; + } + if (!g.moved && g.edge === null && Math.abs(p.x - g.pointerX) < DRAG_START) return; + g.moved = true; + const before = this.committedSpan(wave); + if (!before) return; + const at = xToSeconds(this.view, p.x); + g.span = draggedSpan(before, g.edge, g.from, at, wave.duration, LEAST_SPAN); + this.redraw(); + } + + private onPointerUp(): void { + const g = this.gesture; + this.gesture = null; + if (g?.kind !== 'span' || !g.moved) return this.redraw(); + const asset = this.asset(); + const wave = this.wave(); + if (asset && wave) this.commit(asset, g.span, wave, g.edge ? 'Move sound cut' : 'Cut sound'); + this.redraw(); + } + + private committedSpan(wave: Wave): CutSpan | null { + const asset = this.asset(); + if (!asset) return null; + const cut = cutOf(resolveAssetProps(this.state.project, asset)); + return spanOf(cut.start, cut.end, wave.duration); + } + + /** Writes a span as the asset's start and end, both in one step; nothing where neither changes. */ + private commit(asset: SoundAsset, span: CutSpan, wave: Wave, label: string): void { + const next = storedCut(span, wave.duration, wave.rate); + const now = cutOf(resolveAssetProps(this.state.project, asset)); + const commands = []; + if (next.start !== now.start) commands.push(new SetAssetPropertyCommand(asset.id, START_KEY, next.start)); + if (next.end !== now.end) commands.push(new SetAssetPropertyCommand(asset.id, END_KEY, next.end)); + if (commands.length > 0) this.state.execute(new CompoundCommand(label, commands)); + } + + private onWheel(e: FederatedWheelEvent): void { + const wave = this.wave(); + if (!wave) return; + const p = this.toLocal(e.global); + this.view = clampWave(zoomWave(this.view, zoomFactor(e.deltaY), p.x), wave.duration, this.w); + this.needsFit = false; + this.redraw(); + e.preventDefault(); + } + + private updateCursor(x: number, y: number): void { + const span = this.shownSpan(); + const inside = x >= 0 && x <= this.w && y >= 0 && y <= this.h; + const cursor = inside && span && edgeAt(this.view, span, x) ? 'ew-resize' : 'default'; + if (this.cursor !== cursor) this.cursor = cursor; + } + + override layout(): void { + if (this.needsFit) this.fit(); + } + + override redraw(): void { + const wave = this.wave(); + if (this.needsFit && wave && this.w > 0) return this.fit(); + const c = Theme.colors; + const { w, h } = this; + const wh = this.waveHeight; + this.bg.clear().rect(0, 0, w, h).fill(c.viewport); + this.bg.rect(0, 0, w, RULER_HEIGHT).fill(c.panelHeader); + this.bg.rect(0, RULER_HEIGHT - 1, w, 1).fill(c.border); + this.clip.clear().rect(0, 0, w, h).fill(0xffffff); + this.waveG.clear(); + this.overlay.clear(); + const asset = this.asset(); + const span = this.shownSpan(); + this.hint.visible = !wave; + this.hint.text = !asset + ? 'No sound selected – pick one in the Assets panel.' + : !asset.file + ? 'Not uploaded.' + : soundLibrary.failedToLoad(asset.file) + ? 'Could not be decoded.' + : 'Loading…'; + this.hint.position.set(Theme.padding * 2, RULER_HEIGHT + Theme.padding * 2); + if (!wave || !span || !asset) { + this.syncLabels([]); + return; + } + const x0 = secondsToX(this.view, 0); + const x1 = secondsToX(this.view, wave.duration); + const mid = RULER_HEIGHT + wh / 2; + const half = Math.max(1, wh / 2 - 6); + // The file itself a shade lighter than what lies before and after it. + const left = Math.max(0, x0); + const right = Math.min(w, x1); + if (right > left) this.bg.rect(left, RULER_HEIGHT, right - left, wh).fill({ color: c.panel, alpha: 0.5 }); + if (right > left) this.waveG.rect(left, mid, right - left, 1).fill({ color: c.border }); + this.drawWave(wave, left, right, mid, half); + this.drawRuler(left, right); + this.drawCut(asset, wave, span, mid, half); + if (this.playhead !== null) { + const x = secondsToX(this.view, this.playhead); + if (x >= 0 && x <= w) this.overlay.rect(x - 0.5, 0, 1, h).fill(c.hot); + } + } + + /** A bar from the lowest to the highest sample per pixel, or the samples themselves zoomed far in. */ + private drawWave(wave: Wave, left: number, right: number, mid: number, half: number): void { + const color = { color: 0xffffff, alpha: 0.75 }; + const perPx = wave.rate / this.view.pxPerSecond; + if (perPx < 1) { + const first = Math.max(0, Math.floor(xToSeconds(this.view, left) * wave.rate)); + const last = Math.min( + wave.channels[0]!.length - 1, + Math.ceil(xToSeconds(this.view, right) * wave.rate), + ); + const points: number[] = []; + for (let i = first; i <= last; i++) { + const v = wave.channels.reduce( + (most, data) => (Math.abs(data[i]!) > Math.abs(most) ? data[i]! : most), + 0, + ); + points.push(secondsToX(this.view, i / wave.rate), mid - Math.max(-1, Math.min(1, v)) * half); + } + if (points.length >= 4) this.waveG.poly(points, false).stroke({ width: 1, ...color }); + return; + } + for (let x = Math.floor(left); x < right; x++) { + const from = xToSeconds(this.view, x) * wave.rate; + const to = xToSeconds(this.view, x + 1) * wave.rate; + const { min, max } = spanPeak(wave.channels, wave.peaks, from, to); + const top = mid - Math.min(1, max) * half; + const bottom = mid - Math.max(-1, min) * half; + this.waveG.rect(x, top, 1, Math.max(1, bottom - top)).fill(color); + } + } + + private drawRuler(left: number, right: number): void { + const c = Theme.colors; + const step = rulerStep(this.view.pxPerSecond, LABEL_SPACING); + const placed: { text: string; x: number }[] = []; + const first = Math.ceil(Math.max(0, xToSeconds(this.view, left)) / step); + for (let k = first; ; k++) { + const seconds = k * step; + const x = secondsToX(this.view, seconds); + if (x > right + 0.5) break; + this.overlay.rect(x, RULER_HEIGHT - 8, 1, 7).fill({ color: c.textMuted, alpha: 0.8 }); + placed.push({ text: rulerLabel(seconds, step), x: x + 3 }); + } + this.syncLabels(placed); + } + + /** Dark outside the stretch, a handle at each edge, and the ramp at each edge inside the file. */ + private drawCut(asset: SoundAsset, wave: Wave, span: CutSpan, mid: number, half: number): void { + const c = Theme.colors; + const { h } = this; + const a = secondsToX(this.view, span.start); + const b = secondsToX(this.view, span.end); + const shade = { color: 0x000000, alpha: 0.55 }; + if (a > 0) this.overlay.rect(0, RULER_HEIGHT, a, h - RULER_HEIGHT).fill(shade); + if (b < this.w) this.overlay.rect(b, RULER_HEIGHT, this.w - b, h - RULER_HEIGHT).fill(shade); + this.overlay.rect(a, 0, Math.max(1, b - a), 4).fill({ color: c.accent, alpha: 0.6 }); + const fade = cutOf(resolveAssetProps(this.state.project, asset)).fade; + const range = cutRange(wave.channels[0]!.length, wave.rate, { ...span, fade }); + const top = mid - half; + const ramp = (from: number, to: number, rising: boolean) => { + const xa = secondsToX(this.view, from); + const xb = secondsToX(this.view, to); + if (xb - xa < 2) return; + this.overlay + .moveTo(xa, rising ? mid + half : top) + .lineTo(xb, rising ? top : mid + half) + .stroke({ width: 1, color: c.hot, alpha: 0.7 }); + }; + if (range.fadeIn > 0) ramp(span.start, span.start + range.fadeIn / wave.rate, true); + if (range.fadeOut > 0) ramp(span.end - range.fadeOut / wave.rate, span.end, false); + for (const [x, edge] of [ + [a, 'start'], + [b, 'end'], + ] as const) { + const held = this.gesture?.kind === 'span' && this.gesture.edge === edge; + this.overlay.rect(x - 1, 0, 2, h).fill(held ? c.hot : c.accent); + const toward = edge === 'start' ? 1 : -1; + this.overlay + .moveTo(x, 0) + .lineTo(x + toward * 8, 0) + .lineTo(x, 10) + .closePath() + .fill(held ? c.hot : c.accent); + } + } + + private syncLabels(placed: readonly { text: string; x: number }[]): void { + placed.forEach((label, i) => { + let text = this.texts[i]; + if (!text) { + text = crispText({ style: textStyle('small') }); + this.texts.push(text); + this.labels.addChild(text); + } + text.visible = true; + if (text.text !== label.text) text.text = label.text; + text.position.set(label.x, 6); + }); + for (let i = placed.length; i < this.texts.length; i++) this.texts[i]!.visible = false; + } +} diff --git a/src/editor/sound/waveView.ts b/src/editor/sound/waveView.ts new file mode 100644 index 00000000..51fd7a0c --- /dev/null +++ b/src/editor/sound/waveView.ts @@ -0,0 +1,188 @@ +import { clamp } from '../../ui'; + +/** + * The time axis of the Sound tab (#343), apart from Pixi so a test can drive it: seconds run left to + * right, `x` is measured from the left edge of the area that shows the wave. + */ +export interface WaveView { + /** Pixels from the area's left edge to second 0. */ + panX: number; + pxPerSecond: number; +} + +/** From a few minutes across a screen to a sample every few pixels at 48 kHz. */ +export const MIN_PX_PER_SECOND = 0.5; +export const MAX_PX_PER_SECOND = 200_000; +/** How far either end of the file can be panned in from its edge of the area, as a share of its width. */ +const LEAD = 0.5; +/** Pixels either side of a handle that grab it. */ +export const EDGE_REACH = 6; + +export function secondsToX(view: WaveView, seconds: number): number { + return view.panX + seconds * view.pxPerSecond; +} + +export function xToSeconds(view: WaveView, x: number): number { + return (x - view.panX) / view.pxPerSecond; +} + +/** Zooms about `x`, so the second under the pointer stays under it. */ +export function zoomWave(view: WaveView, factor: number, x: number): WaveView { + const pxPerSecond = clamp(view.pxPerSecond * factor, MIN_PX_PER_SECOND, MAX_PX_PER_SECOND); + return { panX: x - xToSeconds(view, x) * pxPerSecond, pxPerSecond }; +} + +/** The whole file across `width`, a little room either side. */ +export function fitWave(duration: number, width: number, margin: number): WaveView { + const room = Math.max(1, width - margin * 2); + const pxPerSecond = clamp(room / Math.max(1e-6, duration), MIN_PX_PER_SECOND, MAX_PX_PER_SECOND); + return { panX: margin, pxPerSecond }; +} + +/** + * As the music tabs have it across and down: the start of the file no further in than half the + * width, its end no further in from the right – and a file that fits is not pushed off the left. + */ +export function clampWave(view: WaveView, duration: number, width: number): WaveView { + const lead = width * LEAD; + const least = Math.min(0, width - lead - duration * view.pxPerSecond); + return { ...view, panX: clamp(view.panX, least, lead) }; +} + +/** Seconds between the ruler's labels: 1, 2 or 5 times a power of ten, at least `minPx` apart. */ +export function rulerStep(pxPerSecond: number, minPx: number): number { + const rough = minPx / pxPerSecond; + const power = 10 ** Math.floor(Math.log10(rough)); + for (const step of [1, 2, 5, 10]) if (step * power >= rough) return step * power; + return 10 * power; +} + +/** A second as the ruler writes it, with as many decimals as its step needs. */ +export function rulerLabel(seconds: number, step: number): string { + const decimals = Math.max(0, Math.min(6, -Math.floor(Math.log10(step) + 1e-9))); + if (seconds >= 60) { + const minutes = Math.floor(seconds / 60); + const rest = (seconds - minutes * 60).toFixed(decimals); + return `${minutes}:${rest.padStart(decimals > 0 ? decimals + 3 : 2, '0')}`; + } + return `${seconds.toFixed(decimals)}s`; +} + +/** The stretch a sound plays, in seconds of its file: `end` is never 0 here. */ +export interface CutSpan { + start: number; + end: number; +} + +/** The span an asset's start and end mean in a file of `duration`: an end of 0, or past it, is its end. */ +export function spanOf(start: number, end: number, duration: number): CutSpan { + const to = end > 0 ? Math.min(end, duration) : duration; + const from = clamp(start, 0, Math.max(0, to)); + return { start: from, end: Math.max(from, to) }; +} + +/** Which handle is at `x`, the nearer one where both are in reach. */ +export function edgeAt(view: WaveView, span: CutSpan, x: number): 'start' | 'end' | null { + const toStart = Math.abs(secondsToX(view, span.start) - x); + const toEnd = Math.abs(secondsToX(view, span.end) - x); + if (Math.min(toStart, toEnd) > EDGE_REACH) return null; + return toEnd < toStart || (toEnd === toStart && x > secondsToX(view, span.end)) ? 'end' : 'start'; +} + +/** + * The span a drag leaves: one edge moved to `at`, or with no edge a new span from where it was + * pressed to `at`. Held inside the file and at least `least` seconds long; an edge dragged past the + * other stops there. + */ +export function draggedSpan( + span: CutSpan, + edge: 'start' | 'end' | null, + from: number, + at: number, + duration: number, + least: number, +): CutSpan { + const inFile = (s: number) => clamp(s, 0, duration); + const gap = Math.min(least, duration); + if (edge === 'start') return { start: Math.min(inFile(at), span.end - gap), end: span.end }; + if (edge === 'end') return { start: span.start, end: Math.max(inFile(at), span.start + gap) }; + const [a, b] = [inFile(from), inFile(at)].sort((p, q) => p - q) as [number, number]; + return b - a >= gap ? { start: a, end: b } : { start: a, end: Math.min(duration, a + gap) }; +} + +/** + * What an asset stores for a span: seconds, an end at the end of the file as 0 – so a span over the + * whole file is no cut at all, and plays the file itself. + */ +export function storedCut(span: CutSpan, duration: number, rate: number): { start: number; end: number } { + const sample = 1 / rate; + const start = span.start < sample / 2 ? 0 : roundTo(span.start, rate); + const end = duration - span.end < sample / 2 ? 0 : roundTo(span.end, rate); + return { start, end }; +} + +/** To the nearest sample, written short. */ +function roundTo(seconds: number, rate: number): number { + return Number((Math.round(seconds * rate) / rate).toFixed(6)); +} + +/** Samples each block of `WavePeaks` covers. */ +export const PEAK_BLOCK = 256; + +/** The lowest and highest sample of each block of a sound, over all its channels. */ +export interface WavePeaks { + min: Float32Array; + max: Float32Array; +} + +export function wavePeaks(channels: readonly Float32Array[]): WavePeaks { + const length = channels[0]?.length ?? 0; + const blocks = Math.ceil(length / PEAK_BLOCK); + const min = new Float32Array(blocks).fill(Infinity); + const max = new Float32Array(blocks).fill(-Infinity); + for (const data of channels) + for (let i = 0; i < data.length; i++) { + const b = (i / PEAK_BLOCK) | 0; + const v = data[i]!; + if (v < min[b]!) min[b] = v; + if (v > max[b]!) max[b] = v; + } + return { min, max }; +} + +/** + * The lowest and highest sample from `from` up to `to` (sample indices, `to` after `from`): read + * from the blocks where the stretch spans whole ones, from the samples at its ends. + */ +export function spanPeak( + channels: readonly Float32Array[], + peaks: WavePeaks, + from: number, + to: number, +): { min: number; max: number } { + const length = channels[0]?.length ?? 0; + const a = Math.max(0, Math.floor(from)); + const b = Math.min(length, Math.max(a + 1, Math.ceil(to))); + let min = Infinity; + let max = -Infinity; + const raw = (s: number, e: number) => { + for (const data of channels) + for (let i = s; i < e; i++) { + const v = data[i]!; + if (v < min) min = v; + if (v > max) max = v; + } + }; + const firstBlock = Math.ceil(a / PEAK_BLOCK); + const lastBlock = Math.floor(b / PEAK_BLOCK); + if (lastBlock <= firstBlock) raw(a, b); + else { + raw(a, firstBlock * PEAK_BLOCK); + for (let k = firstBlock; k < lastBlock; k++) { + if (peaks.min[k]! < min) min = peaks.min[k]!; + if (peaks.max[k]! > max) max = peaks.max[k]!; + } + raw(lastBlock * PEAK_BLOCK, b); + } + return min <= max ? { min, max } : { min: 0, max: 0 }; +} diff --git a/src/editor/soundfx/SoundFxEditor.ts b/src/editor/soundfx/SoundFxEditor.ts index 65c8023e..70785caf 100644 --- a/src/editor/soundfx/SoundFxEditor.ts +++ b/src/editor/soundfx/SoundFxEditor.ts @@ -1,5 +1,6 @@ import type { Asset, SoundFxAsset } from '../../core/model/Asset'; import { findAssetOf } from '../../core/model/Project'; +import { playedSoundOf } from '../../core/properties/playedSound'; import { resolveAssetProps } from '../../core/properties/resolve'; import { volumeOf } from '../../core/properties/sounds'; import { soundFxBuildFor, soundFxParams } from '../../core/soundfx/compile'; @@ -101,7 +102,7 @@ export class SoundFxEditor extends AssetTabEditor { const plan = asset && soundFxBuildFor(asset).compiled; if (!sound?.file || !plan) return; const volume = volumeOf(resolveAssetProps(this.state.project, sound)); - const voice = soundLibrary.play(sound.file, volume, false, PREVIEW_OWNER, { + const voice = soundLibrary.play(playedSoundOf(this.state.project, sound), volume, false, PREVIEW_OWNER, { plan, params: { ...this.values }, }); diff --git a/src/editor/state.ts b/src/editor/state.ts index 59cd273a..f7a1470b 100644 --- a/src/editor/state.ts +++ b/src/editor/state.ts @@ -803,6 +803,13 @@ export class EditorState { return findAssetOf(this.project, s.assetId, 'curve')?.id ?? null; } + /** The sound the selection refers to (#343); the stretch it is cut to is set in its tab. */ + get selectedSoundId(): string | null { + const s = this.selection; + if (s?.kind !== 'asset') return null; + return findAssetOf(this.project, s.assetId, 'sound')?.id ?? null; + } + /** The pattern the selection refers to (#333); only the asset itself, its notes are picked in the tab. */ get selectedPatternId(): string | null { const s = this.selection; diff --git a/src/runtime/systems/behaviorContext.ts b/src/runtime/systems/behaviorContext.ts index aa6fb0a8..2d671ac3 100644 --- a/src/runtime/systems/behaviorContext.ts +++ b/src/runtime/systems/behaviorContext.ts @@ -11,7 +11,7 @@ import { JOINT_CONNECTED_KEY } from '../../core/model/joint'; import { objectReference } from '../../core/model/objectRefs'; import { findAsset, findAssetOf, findScene, type Project } from '../../core/model/Project'; import { renderLayerAlong } from '../../core/model/renderLayers'; -import { soundFileOf } from '../../core/model/sound'; +import { playedSoundById, playedSoundOf } from '../../core/properties/playedSound'; import { hasRenderLayer } from '../../core/properties/PropertySchema'; import { resolveAssetProps } from '../../core/properties/resolve'; import { loopsOf, volumeOf } from '../../core/properties/sounds'; @@ -193,14 +193,14 @@ export function behaviorContext( stopSong: (songId, fadeOut) => run.audio.stopSequence(songVoiceName(songId), false, heldFade(fadeOut)), stopPlayback: (playback, fadeOut) => run.audio.stopPlayback(playback, heldFade(fadeOut)), soundLength: (assetId) => { - const file = soundFileOf(project, assetId); + const file = playedSoundById(project, assetId); return file ? run.audio.length(file) : 0; }, // An id nothing resolves stops nothing, rather than silencing the whole run by accident; // only the empty id means "everything", which is what the node promises. soundTail: (voice) => run.audio.tail(voice), stopSound: (assetId, letRing) => { - const file = assetId === '' ? '' : soundFileOf(project, assetId); + const file = playedSoundById(project, assetId); if (assetId === '' || file) run.audio.stop(file, letRing); }, stopVoice: (voice, letRing) => run.audio.stopVoice(voice, letRing), @@ -270,7 +270,7 @@ function playSound( const fx = fxId ? findAssetOf(project, fxId, 'soundfx') : undefined; const plan = fx && soundFxBuildFor(fx).compiled; return audio.play( - asset.file, + playedSoundOf(project, asset), level, loop ?? loopsOf(props), owner, @@ -404,7 +404,7 @@ export function playClip( Math.min(dbToGain(SOUND_ITEM_MAX_DB), volumeOf(resolveAssetProps(project, asset)) * clip.volume), ); return audio.playClip( - asset.file, + playedSoundOf(project, asset), { at: clip.at, offset: clip.offset, duration: clip.duration, volume, bus: clip.bus }, clipFx(project, clip.fx, clip.tempo), ); diff --git a/src/viewport/SoundLibrary.ts b/src/viewport/SoundLibrary.ts index 7b921f2e..4fdea220 100644 --- a/src/viewport/SoundLibrary.ts +++ b/src/viewport/SoundLibrary.ts @@ -21,6 +21,7 @@ import { type BusLevel, Playbacks, type SequencePlay } from '../core/music/playb import type { ScheduledClip, ScheduledNote } from '../core/music/Scheduler'; import { dbToGain, SOUND_ITEM_MAX_DB } from '../core/music/song'; import { DEFAULT_TEMPO } from '../core/music/time'; +import { type CutRange, cutChannels, cutRange, parseCutName } from '../core/model/sound'; import type { AudioSink, ClipTiming, NoteTiming } from '../runtime/systems/Audio'; import type { PitchOptions } from './pitchWorklet'; import pitchWorklet from './pitchWorklet.ts?worker&url'; @@ -39,6 +40,8 @@ const DECODE_RATE = 48000; const FADE = 0.01; /** How finely `peaks` cuts a sound up. */ const PEAK_BUCKETS = 1024; +/** Samples the cuts made so far may hold together before the oldest are let go; one is always kept. */ +const CUT_BUDGET = 64 * 1024 * 1024; /** * One number of a voice that can still change while it plays (#315): a level, a share of a Mix, the @@ -221,6 +224,12 @@ export class SoundLibrary implements AudioSink { private context: AudioContext | null = null; private offline: OfflineAudioContext | null = null; private readonly buffers = new Map(); + /** + * The stretches of files that sound assets are cut to (#343), made from the whole file on first + * play, oldest first. Unlike a file they can be made again at any time, so they are let go past a + * budget: a cut dragged about in its tab would otherwise leave a copy of the sound per try. + */ + private readonly cuts = new Map(); /** The pitch worklet being loaded, and whether it is there once it is known; null before that. */ private pitching: Promise | null = null; private canPitch: boolean | null = null; @@ -286,7 +295,7 @@ export class SoundLibrary implements AudioSink { * same frame it asked for it, and the voice then starts with those values. */ play(file: string, volume: number, loop: boolean, owner = '', fx: FxVoice = DRY_VOICE): number { - if (file === '' || this.failed.has(file)) return 0; + if (file === '' || this.hasFailed(file)) return 0; const layered = { layers: [fx.plan], params: fx.params, tempo: fx.tempo ?? DEFAULT_TEMPO }; return this.begin({ id: 0, file, files: [file], volume, loop, owner, fx: layered, duration: 0, at: 0 }); } @@ -307,7 +316,7 @@ export class SoundLibrary implements AudioSink { timing: NoteTiming = {}, ): number { const files = [...new Set(layers.flatMap(filesOf))].filter( - (file) => file !== '' && !this.failed.has(file), + (file) => file !== '' && !this.hasFailed(file), ); if (files.length === 0) return 0; const fx = { layers, params, tempo: timing.tempo ?? DEFAULT_TEMPO }; @@ -322,7 +331,7 @@ export class SoundLibrary implements AudioSink { * gets its bytes late starts as far into the sound as it would be by then. */ playClip(file: string, clip: ClipTiming, fx: FxVoice = DRY_VOICE): number { - if (file === '' || this.failed.has(file)) return 0; + if (file === '' || this.hasFailed(file)) return 0; const layered = { layers: [fx.plan], params: fx.params, tempo: fx.tempo ?? DEFAULT_TEMPO }; const { at, offset, duration, volume, bus } = clip; return this.begin({ @@ -537,7 +546,7 @@ export class SoundLibrary implements AudioSink { private begin(start: Start): number { const id = this.nextId++; const asked: Start = { ...start, id }; - const loaded = start.files.every((file) => this.buffers.has(file)); + const loaded = start.files.every((file) => this.buffer(file) !== null); const pitching = start.fx.layers.some(pitches) ? this.pitchWorklet() : null; if (loaded && (!pitching || this.canPitch !== null)) { this.start(asked); @@ -598,6 +607,11 @@ export class SoundLibrary implements AudioSink { return voice ? this.tailOf(voice.fx) : 0; } + /** The second of the audio clock a voice started at; null while it waits on its bytes or is gone. */ + startedAt(id: number): number | null { + return this.voices.find((voice) => voice.id === id)?.at ?? null; + } + /** Whether a voice is still heard – or waiting on its bytes to be. A pattern or song counts as one. */ playing(id: number): boolean { return this.sounding(id) || this.sequences.has(id); @@ -743,9 +757,23 @@ export class SoundLibrary implements AudioSink { return pitchWorklet; } - /** How long a decoded sound runs, in seconds; null while it has not been loaded (test hook). */ + /** + * How long a decoded sound runs, in seconds – a cut (#343) as long as what it keeps; null while it + * has not been loaded (test hook). + */ duration(file: string): number | null { - return this.buffers.get(file)?.duration ?? null; + const range = this.rangeOf(file); + return range ? (range.to - range.from) / range.whole.sampleRate : null; + } + + /** The decoded samples of a file as it is, without a cut; null while it has not been loaded. */ + decoded(file: string): AudioBuffer | null { + return this.buffers.get(file) ?? null; + } + + /** Whether a file could not be fetched or decoded – it is not tried again. */ + failedToLoad(file: string): boolean { + return this.hasFailed(file); } length(file: string): number { @@ -759,13 +787,14 @@ export class SoundLibrary implements AudioSink { peaks(file: string): Float32Array | null { const known = this.peaked.get(file); if (known) return known; - const buffer = this.buffers.get(file); - if (!buffer) return null; + const range = this.rangeOf(file); + if (!range) return null; + const { whole, from, to } = range; const peaks = new Float32Array(PEAK_BUCKETS); - for (let c = 0; c < buffer.numberOfChannels; c++) { - const data = buffer.getChannelData(c); - for (let i = 0; i < data.length; i++) { - const bucket = Math.floor((i / data.length) * PEAK_BUCKETS); + for (let c = 0; c < whole.numberOfChannels; c++) { + const data = whole.getChannelData(c); + for (let i = from; i < to; i++) { + const bucket = Math.floor(((i - from) / (to - from)) * PEAK_BUCKETS); peaks[bucket] = Math.max(peaks[bucket]!, Math.abs(data[i]!)); } } @@ -782,8 +811,15 @@ export class SoundLibrary implements AudioSink { for (const file of files) if (file) void this.load(file); } - /** Fetches and decodes once per file; a second play while it is in flight waits on the first. */ + /** + * Fetches and decodes once per file; a second play while it is in flight waits on the first. A cut + * (#343) loads its file and is made from it. + */ async load(file: string): Promise { + if (parseCutName(file).cut) { + const whole = await this.load(parseCutName(file).file); + return whole ? this.buffer(file) : null; + } const held = this.buffers.get(file); if (held) return held; if (this.failed.has(file)) return null; @@ -934,7 +970,13 @@ export class SoundLibrary implements AudioSink { const wiring: Wiring = { context, at: when, - buffers: this.buffers, + // Held here, not looked up later: a cut can be let go while its voice plays. + buffers: new Map( + files.flatMap((name) => { + const buffer = this.buffer(name); + return buffer ? [[name, buffer] as const] : []; + }), + ), file, loop, rings, @@ -957,7 +999,7 @@ export class SoundLibrary implements AudioSink { layers: fx.layers.length, pitches: wiring.pitches, stages: wiring.stages, - duration: Math.max(0, ...files.map((name) => this.buffers.get(name)?.duration ?? 0)), + duration: Math.max(0, ...files.map((name) => this.duration(name) ?? 0)), at: when, envelopes: wiring.envelopes.size, }; @@ -1173,6 +1215,50 @@ export class SoundLibrary implements AudioSink { return (this.context ??= new AudioContext()); } + /** What a name plays, a cut made from its file the first time; null while the file is not loaded. */ + private buffer(name: string): AudioBuffer | null { + const whole = this.buffers.get(name); + if (whole) return whole; + const held = this.cuts.get(name); + if (held) { + this.cuts.delete(name); + this.cuts.set(name, held); + return held; + } + const range = this.rangeOf(name); + if (!range) return null; + const made = this.decoder().createBuffer( + range.whole.numberOfChannels, + range.to - range.from, + range.whole.sampleRate, + ); + const channels = Array.from({ length: made.numberOfChannels }, (_, c) => range.whole.getChannelData(c)); + cutChannels(channels, range).forEach((data, c) => made.copyToChannel(data, c)); + this.cuts.set(name, made); + const samples = (buffer: AudioBuffer) => buffer.length * buffer.numberOfChannels; + let total = [...this.cuts.values()].reduce((sum, buffer) => sum + samples(buffer), 0); + for (const [other, buffer] of this.cuts) { + if (total <= CUT_BUDGET || other === name) break; + this.cuts.delete(other); + total -= samples(buffer); + } + return made; + } + + /** The samples of its loaded file a name plays: all of them, or the stretch a cut keeps. */ + private rangeOf(name: string): (CutRange & { whole: AudioBuffer }) | null { + const { file, cut } = parseCutName(name); + const whole = this.buffers.get(file); + if (!whole) return null; + if (!cut) return { whole, from: 0, to: whole.length, fadeIn: 0, fadeOut: 0 }; + return { whole, ...cutRange(whole.length, whole.sampleRate, cut) }; + } + + /** Whether the file of a name could not be had: a cut of it cannot either. */ + private hasFailed(name: string): boolean { + return this.failed.has(parseCutName(name).file); + } + /** Loads the pitch worklet into the context once; says whether it is there. */ private pitchWorklet(): Promise { return (this.pitching ??= (async () => { diff --git a/tests/core/sounds.test.ts b/tests/core/sounds.test.ts index bcf34fc3..91dad7c2 100644 --- a/tests/core/sounds.test.ts +++ b/tests/core/sounds.test.ts @@ -11,10 +11,27 @@ import { } from '../../src/core/commands/assetCommands'; import type { BehaviorAsset, SoundAsset } from '../../src/core/model/Asset'; import { isPlaceableType } from '../../src/core/model/AssetTypes'; -import { isSoundFile, soundFileOf } from '../../src/core/model/sound'; +import { + cutChannels, + cutName, + cutRange, + isSoundFile, + parseCutName, + soundFileOf, +} from '../../src/core/model/sound'; import { assetRefChoices } from '../../src/core/properties/assetRef'; import { schemaFor } from '../../src/core/properties/PropertySchema'; -import { LOOP_KEY, loopsOf, VOLUME_KEY, volumeOf } from '../../src/core/properties/sounds'; +import { playedSoundById, playedSoundOf } from '../../src/core/properties/playedSound'; +import { + cutOf, + END_KEY, + FADE_KEY, + LOOP_KEY, + loopsOf, + START_KEY, + VOLUME_KEY, + volumeOf, +} from '../../src/core/properties/sounds'; import { deserializeProject } from '../../src/core/store/serialize'; import type { AudioSink } from '../../src/runtime/systems/Audio'; import { @@ -99,7 +116,14 @@ describe('the sound asset', () => { expect(asset).toMatchObject({ type: 'sound', file: '' }); expect(volumeOf(asset.props)).toBe(1); expect(loopsOf(asset.props)).toBe(false); - expect(schemaFor('sound').map((def) => def.key)).toEqual([VOLUME_KEY, LOOP_KEY]); + expect(schemaFor('sound').map((def) => def.key)).toEqual([ + VOLUME_KEY, + LOOP_KEY, + START_KEY, + END_KEY, + FADE_KEY, + ]); + expect(cutOf(asset.props)).toEqual({ start: 0, end: 0, fade: 0.01 }); }); it('refuses a file name the server could not have produced', () => { @@ -142,6 +166,62 @@ describe('the sound asset', () => { }); }); +describe('the cut of a sound (#343)', () => { + it('is no cut at all over the whole file, and a name of its own otherwise', () => { + const whole = { start: 0, end: 0, fade: 0.01 }; + expect(cutName(FILE, whole)).toBe(FILE); + expect(cutName('', { ...whole, start: 1 })).toBe(''); + const name = cutName(FILE, { start: 0.25, end: 1.5, fade: 0.01 }); + expect(name).toBe(`${FILE}#0.25~1.5~0.01`); + expect(parseCutName(name)).toEqual({ file: FILE, cut: { start: 0.25, end: 1.5, fade: 0.01 } }); + expect(parseCutName(FILE)).toEqual({ file: FILE, cut: null }); + expect(parseCutName(`${FILE}#x~-2`)).toEqual({ file: FILE, cut: { start: 0, end: 0, fade: 0 } }); + }); + + it('keeps at least a sample inside the file, and ramps only the edges inside it', () => { + // 10 samples at 10 Hz: one sample per tenth of a second. + expect(cutRange(10, 10, { start: 0.2, end: 0.8, fade: 0.2 })).toEqual({ + from: 2, + to: 8, + fadeIn: 2, + fadeOut: 2, + }); + expect(cutRange(10, 10, { start: 0, end: 0.5, fade: 0.2 })).toMatchObject({ fadeIn: 0, fadeOut: 2 }); + expect(cutRange(10, 10, { start: 0.5, end: 0, fade: 0.2 })).toMatchObject({ + to: 10, + fadeIn: 2, + fadeOut: 0, + }); + // Past the end, an end before the start, and a fade longer than half of what is kept. + expect(cutRange(10, 10, { start: 5, end: 0, fade: 0 })).toMatchObject({ from: 9, to: 10 }); + expect(cutRange(10, 10, { start: 0.6, end: 0.3, fade: 0 })).toMatchObject({ from: 6, to: 7 }); + expect(cutRange(10, 10, { start: 0.2, end: 0.6, fade: 1 })).toMatchObject({ fadeIn: 2, fadeOut: 2 }); + }); + + it('cuts every channel alike, rising from silence and falling to it at an inner edge', () => { + const left = new Float32Array(10).fill(1); + const right = new Float32Array(10).fill(-0.5); + const [l, r] = cutChannels([left, right], { from: 2, to: 8, fadeIn: 2, fadeOut: 2 }); + expect(Array.from(l!)).toEqual([0, 0.5, 1, 1, 0.5, 0]); + expect(Array.from(r!)).toEqual([-0, -0.25, -0.5, -0.5, -0.25, -0]); + expect(Array.from(left)).toEqual(new Array(10).fill(1)); + }); + + it('is what every way of playing the asset is given, its fade in milliseconds on the asset', () => { + const f = playingScene(); + expect(playedSoundOf(f.project, f.sound)).toBe(FILE); + f.stack.execute(new SetAssetPropertyCommand(f.sound.id, START_KEY, 0.5)); + f.stack.execute(new SetAssetPropertyCommand(f.sound.id, FADE_KEY, 25)); + const name = `${FILE}#0.5~0~0.025`; + expect(playedSoundOf(f.project, f.sound)).toBe(name); + expect(playedSoundById(f.project, f.sound.id)).toBe(name); + expect(playedSoundById(f.project, '')).toBe(''); + const audio = fakeAudio(); + startWorld(f, { behaviors: true, audio }).frame(); + expect(audio.played).toMatchObject([{ file: name }]); + }); +}); + describe('Play sound', () => { it('plays the asset as the project describes it', () => { const f = playingScene(); diff --git a/tests/editor/wave-view.test.ts b/tests/editor/wave-view.test.ts new file mode 100644 index 00000000..85bff32f --- /dev/null +++ b/tests/editor/wave-view.test.ts @@ -0,0 +1,101 @@ +/** + * The Sound tab's view (#343): seconds across, zoomed about the pointer, and the stretch of the + * file between two handles – how a drag moves it and what the asset then stores. + */ +import { describe, expect, it } from 'vitest'; +import { + clampWave, + draggedSpan, + edgeAt, + fitWave, + rulerLabel, + rulerStep, + secondsToX, + spanOf, + spanPeak, + storedCut, + wavePeaks, + xToSeconds, + zoomWave, +} from '../../src/editor/sound/waveView'; + +describe('the wave view', () => { + it('zooms about the pointer and fits the file with a margin either side', () => { + const view = { panX: 10, pxPerSecond: 100 }; + const zoomed = zoomWave(view, 2, 210); + expect(xToSeconds(zoomed, 210)).toBeCloseTo(xToSeconds(view, 210)); + expect(zoomed.pxPerSecond).toBe(200); + const fitted = fitWave(4, 832, 16); + expect(secondsToX(fitted, 0)).toBe(16); + expect(secondsToX(fitted, 4)).toBe(816); + }); + + it('keeps either end of the file from going further in than half the width', () => { + expect(clampWave({ panX: 900, pxPerSecond: 100 }, 20, 1000).panX).toBe(500); + expect(clampWave({ panX: -1600, pxPerSecond: 100 }, 20, 1000).panX).toBe(-1500); + expect(clampWave({ panX: -900, pxPerSecond: 100 }, 20, 1000).panX).toBe(-900); + // Two seconds fit: they stay on screen. + expect(clampWave({ panX: 20, pxPerSecond: 100 }, 2, 1000).panX).toBe(20); + expect(clampWave({ panX: -50, pxPerSecond: 100 }, 2, 1000).panX).toBe(0); + }); + + it('labels the ruler in steps of 1, 2 or 5, with the decimals the step needs', () => { + expect(rulerStep(100, 70)).toBe(1); + expect(rulerStep(40, 70)).toBe(2); + expect(rulerStep(1000, 70)).toBeCloseTo(0.1); + expect(rulerStep(10, 70)).toBe(10); + expect(rulerLabel(1.5, 0.5)).toBe('1.5s'); + expect(rulerLabel(2, 1)).toBe('2s'); + expect(rulerLabel(0.25, 0.05)).toBe('0.25s'); + expect(rulerLabel(75, 5)).toBe('1:15'); + }); +}); + +describe('the cut span', () => { + const view = { panX: 0, pxPerSecond: 100 }; + + it('reads an end of 0 or past the file as its end', () => { + expect(spanOf(0, 0, 3)).toEqual({ start: 0, end: 3 }); + expect(spanOf(1, 9, 3)).toEqual({ start: 1, end: 3 }); + expect(spanOf(2.5, 1, 3)).toEqual({ start: 1, end: 1 }); + }); + + it('finds the handle in reach, the nearer one where both are', () => { + const span = { start: 1, end: 2 }; + expect(edgeAt(view, span, 103)).toBe('start'); + expect(edgeAt(view, span, 197)).toBe('end'); + expect(edgeAt(view, span, 150)).toBeNull(); + expect(edgeAt(view, { start: 1, end: 1.04 }, 103)).toBe('end'); + }); + + it('moves one edge, never past the other, or spans a new stretch inside the file', () => { + const span = { start: 1, end: 2 }; + expect(draggedSpan(span, 'start', 1, 0.5, 3, 0.001)).toEqual({ start: 0.5, end: 2 }); + expect(draggedSpan(span, 'start', 1, 2.5, 3, 0.001)).toEqual({ start: 1.999, end: 2 }); + expect(draggedSpan(span, 'end', 2, 9, 3, 0.001)).toEqual({ start: 1, end: 3 }); + expect(draggedSpan(span, null, 2.5, 0.5, 3, 0.001)).toEqual({ start: 0.5, end: 2.5 }); + expect(draggedSpan(span, null, -1, 1, 3, 0.001)).toEqual({ start: 0, end: 1 }); + }); + + it('stores the ends of the file as 0, everything else to the sample', () => { + expect(storedCut({ start: 0, end: 3 }, 3, 48000)).toEqual({ start: 0, end: 0 }); + expect(storedCut({ start: 0.000001, end: 2.999999 }, 3, 48000)).toEqual({ start: 0, end: 0 }); + expect(storedCut({ start: 1.2345678, end: 2 }, 3, 48000)).toEqual({ start: 1.234563, end: 2 }); + }); +}); + +describe('the wave peaks', () => { + it('reads the lowest and highest sample of any stretch, over the blocks and the samples at its ends', () => { + const left = new Float32Array(1000); + const right = new Float32Array(1000); + left[10] = 0.5; + right[600] = -0.75; + left[999] = 0.9; + const peaks = wavePeaks([left, right]); + expect(peaks.max).toHaveLength(4); + expect(spanPeak([left, right], peaks, 0, 1000)).toEqual({ min: -0.75, max: Math.fround(0.9) }); + expect(spanPeak([left, right], peaks, 5, 20)).toEqual({ min: 0, max: 0.5 }); + expect(spanPeak([left, right], peaks, 11, 599)).toEqual({ min: 0, max: 0 }); + expect(spanPeak([left, right], peaks, 300.2, 300.4)).toEqual({ min: 0, max: 0 }); + }); +}); diff --git a/tests/viewport/sound-library.test.ts b/tests/viewport/sound-library.test.ts index 80beae70..397f217f 100644 --- a/tests/viewport/sound-library.test.ts +++ b/tests/viewport/sound-library.test.ts @@ -159,6 +159,7 @@ function fakeBuffer(_channels: number, length: number, sampleRate: number) { sampleRate, duration: length / sampleRate, getChannelData: () => data, + copyToChannel: (from: Float32Array) => data.set(from), }; } @@ -378,6 +379,23 @@ describe('the sound library', () => { expect(sources.filter((s) => s.started)).toHaveLength(2); }); + it('plays the stretch a sound is cut to, made from its file without fetching it again (#343)', async () => { + const library = new SoundLibrary(); + // The file is 1200 samples at 800 Hz: 1.5 s. The cut keeps 0.5 s to 1 s. + const cut = `${FILE}#0.5~1~0.01`; + await play(library, cut); + expect(library.stats()).toMatchObject({ started: 1, decoded: 1, playing: 1 }); + expect((sources[0]?.buffer as { length: number }).length).toBe(400); + expect(library.duration(cut)).toBe(0.5); + expect(library.duration(FILE)).toBe(1.5); + expect(library.peaks(cut)).toHaveLength(1024); + // The file itself plays whole beside it, and a stop by the cut's name reaches only the cut. + await play(library); + expect((sources[1]?.buffer as { length: number }).length).toBe(1200); + library.stop(cut); + expect(library.stats().playing).toBe(1); + }); + it('loads ahead of the first play without opening the device, so that play starts at once', async () => { const opened: string[] = []; vi.stubGlobal( diff --git a/tools/e2e/34-sound.mjs b/tools/e2e/34-sound.mjs index 9607b76d..c1e77f7d 100644 --- a/tools/e2e/34-sound.mjs +++ b/tools/e2e/34-sound.mjs @@ -1152,6 +1152,108 @@ export async function sound(t) { JSON.stringify({ dropped, both, notice: await json('__gm.state.notice') }), ); + // 34h-2. The Sound tab (#343): the drone's 0.75 s as a wave between two handles. Dragging the end + // handle cuts the sound there, a drag beside the handles spans a new stretch, and that + // stretch is what plays: a name of its own, its samples cut to the sample and ramped from + // silence at an edge inside the file. One drag is one undo step. + const droneId = both.find((a) => a.file === drone.file)?.id; + await evaluate(`__gm.state.openAsset(${JSON.stringify(droneId)})`); + const tab = () => json('__gm.soundTab.span()'); + await waitFor(tab, (span) => span !== null); + const tabOpened = { + current: await json('__gm.soundTab.current()'), + span: await tab(), + info: await json('__gm.soundTab.info()'), + }; + check( + 'opening a sound shows its tab: the whole file between the handles', + tabOpened.current === droneId && + tabOpened.span?.start === 0 && + Math.abs(tabOpened.span?.end - 0.75) < 0.001 && + tabOpened.info === '0.75 s', + JSON.stringify(tabOpened), + ); + const droneProps = async () => (await assetsOfType('sound')).find((a) => a.id === droneId)?.props; + let dragFrom = await json('__gm.soundTab.at(0.75)'); + let dragTo = await json('__gm.soundTab.at(0.5)'); + await t.mouse('mouseMoved', dragFrom.x - 1, dragFrom.y); + await t.mouse('mouseMoved', dragFrom.x, dragFrom.y); + const onHandle = await json('__gm.soundTab.cursor()'); + await t.drag(dragFrom.x, dragFrom.y, dragTo.x, dragTo.y); + const shortened = { props: await droneProps(), info: await json('__gm.soundTab.info()') }; + check( + 'dragging the end handle cuts the sound there, the handle showing it can be dragged', + onHandle === 'ew-resize' && + shortened.props?.start === 0 && + Math.abs(shortened.props?.end - 0.5) < 0.01 && + shortened.info === `${shortened.props?.end.toFixed(2)} s of 0.75 s`, + JSON.stringify({ onHandle, ...shortened }), + ); + await press('z', { ctrl: true }); + const undoneCut = await droneProps(); + await press('z', { ctrl: true, shift: true }); + check( + 'one undo puts the whole file back, a redo the cut', + (undoneCut?.end ?? 0) === 0 && Math.abs((await droneProps())?.end - 0.5) < 0.01, + JSON.stringify({ undoneCut, redone: await droneProps() }), + ); + dragFrom = await json('__gm.soundTab.at(0.1)'); + dragTo = await json('__gm.soundTab.at(0.3)'); + await t.drag(dragFrom.x, dragFrom.y, dragTo.x, dragTo.y); + const spanned = await droneProps(); + const cutPlayed = await json(`__gm.soundTab.played(${JSON.stringify(droneId)})`); + const cutSamples = await evaluate(`__gm.soundTab.samples(${JSON.stringify(cutPlayed)})`); + const keptSamples = spanned ? Math.round((spanned.end - spanned.start) * (cutSamples?.rate ?? 0)) : -1; + check( + 'a drag beside the handles spans a new stretch, played by a name of its own, cut to the sample', + Math.abs(spanned?.start - 0.1) < 0.01 && + Math.abs(spanned?.end - 0.3) < 0.01 && + cutPlayed === `${drone.file}#${spanned?.start}~${spanned?.end}~0.01` && + cutSamples?.length === keptSamples, + JSON.stringify({ spanned, cutPlayed, cutSamples, keptSamples }), + ); + check( + 'the cut rises from silence and falls back to it, the default 10 ms at each edge inside the file', + cutSamples?.head[0] === 0 && Math.abs(cutSamples?.tail.at(-1)) < 0.01, + JSON.stringify(cutSamples), + ); + const tabFade = await json('__gm.soundTab.fadeField()'); + await click(tabFade.x, tabFade.y); + await t.typeText('0'); + await press('Enter'); + await sleep(150); + const unfaded = await json(`__gm.soundTab.played(${JSON.stringify(droneId)})`); + check( + 'the header sets the fade at the edges, in milliseconds on the asset', + (await droneProps())?.fade === 0 && unfaded.endsWith('~0'), + JSON.stringify({ props: await droneProps(), unfaded }), + ); + await evaluate('__gm.sound.stopAll()'); + const tabPlay = await json('__gm.soundTab.playButton()'); + await click(tabPlay.x, tabPlay.y); + const tabHeard = await waitFor(stats, (s) => s.last?.file === unfaded); + const tabPlayhead = await waitFor( + () => json('__gm.soundTab.playhead()'), + (seconds) => seconds !== null && seconds > 0.1, + 10, + ); + const tabEnded = await waitFor( + () => json('__gm.soundTab.playing()'), + (on) => on === false, + ); + check( + 'Play plays only the stretch, its playhead running inside it, and stops when it ends', + tabHeard.last?.file === unfaded && + Math.abs(tabHeard.last?.duration - 0.2) < 0.01 && + tabPlayhead !== null && + tabPlayhead >= 0.1 && + tabPlayhead <= 0.3 && + tabEnded === false && + (await json('__gm.soundTab.playhead()')) === null, + JSON.stringify({ last: tabHeard.last, tabPlayhead, tabEnded }), + ); + await shot('34-sound-tab'); + // 34i. The step takes back everything it made, so the steps after it see the fixture again. await mutate(` const gone = new Set(['sn-logic', ...${JSON.stringify(both.map((a) => a.id))}]);