Cut a sound to a stretch of its file in its own Sound tab (#343)
game-maker / gate (push) Successful in 3m32s
game-maker / suite (push) Failing after 18m20s

A sound asset now opens in a tab that shows its file as a waveform.
Two handles set where the sound starts and ends in the file, and that
stretch is the sample: Play sound, a song item and an instrument's
Sample all play it and nothing else. The file itself is never changed.

- The asset has three new rows: Start (s), End (s) and Edge fade (ms).
  End 0 is the end of the file, the fade is 10 ms by default. Older
  assets have no values and play the whole file as before.
- Every way of playing asks playedSoundOf() for a name instead of the
  file: the file, or <file>#<start>~<end>~<fade> for a cut.
- SoundLibrary makes the cut from the decoded file the first time that
  name plays and ramps each edge inside the file over the fade. Length
  and peaks of a cut are read from the file without a copy.
- Cut buffers can be made again at any time, so the library lets the
  oldest go past 64 Mi samples. A voice holds its buffers itself.
- The tab draws min/max per pixel from 256-sample block peaks, and the
  samples as a line when zoomed in far. Wheel zooms, the middle button
  pans, as in the music tabs. Wave, cut and ruler sit under a mask,
  so a handle panned out of view does not draw over the panels beside.
- Dragging a handle moves that edge, dragging elsewhere spans a new
  stretch. One drag is one undo step for both edges. Whole file puts
  the handles back, Play or Space plays the stretch with a playhead.
- The inspector preview and the Sound FX preview play the cut too, and
  the song lanes draw and repeat it.

Tests:
- Unit: sounds 20, wave-view 8 (new), sound-library 68, plus 258 in
  the touched areas (instrument, song, pattern, inspector, tabs)
- e2e 34 on melektaus-atx: ALL PASSED, 57 checks (7 new)
- Gate on melektaus-atx: CHECK EXIT 0, 253 files, 3936 tests

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
adminandClaude Opus 5.5 committed 2026-10-06 21:42:21 +02:00
1 parent 2fdf97f7de
commit f1f4a57589
24 files changed
+1442 -53

No files matched your search

+6 -2
View File
@@ -15,6 +15,7 @@ import type { InstrumentAsset } from '../model/Asset';
import { graphLookup } from '../model/graphLookup'; import { graphLookup } from '../model/graphLookup';
import { textOf } from '../model/Properties'; import { textOf } from '../model/Properties';
import { findAssetOf, type Project } from '../model/Project'; import { findAssetOf, type Project } from '../model/Project';
import { playedSoundOf } from '../properties/playedSound';
import { resolveAssetProps } from '../properties/resolve'; import { resolveAssetProps } from '../properties/resolve';
import { volumeOf } from '../properties/sounds'; import { volumeOf } from '../properties/sounds';
import { soundFxBuildFor } from '../soundfx/compile'; import { soundFxBuildFor } from '../soundfx/compile';
@@ -71,7 +72,10 @@ export function instrumentSourcesOf(project: Project): InstrumentSources {
lookup: graphLookup(project), lookup: graphLookup(project),
sound: (assetId) => { sound: (assetId) => {
const asset = findAssetOf(project, assetId, 'sound'); const asset = findAssetOf(project, assetId, 'sound');
return asset ? { file: asset.file, volume: volumeOf(resolveAssetProps(project, asset)) } : null; if (!asset) return null;
// An asset without a file stays without one, so the Sample can say so.
const file = asset.file ? playedSoundOf(project, asset) : '';
return { file, volume: volumeOf(resolveAssetProps(project, asset)) };
}, },
soundFx: (assetId) => { soundFx: (assetId) => {
const asset = findAssetOf(project, assetId, 'soundfx'); const asset = findAssetOf(project, assetId, 'soundfx');
@@ -79,7 +83,7 @@ export function instrumentSourcesOf(project: Project): InstrumentSources {
}, },
key: (assetId) => { key: (assetId) => {
const sound = findAssetOf(project, assetId, 'sound'); const sound = findAssetOf(project, assetId, 'sound');
if (sound) return `${sound.file}@${volumeOf(resolveAssetProps(project, sound))}`; if (sound) return `${playedSoundOf(project, sound)}@${volumeOf(resolveAssetProps(project, sound))}`;
const fx = findAssetOf(project, assetId, 'soundfx'); const fx = findAssetOf(project, assetId, 'soundfx');
return fx ? graphFingerprint(fx.graph) : ''; return fx ? graphFingerprint(fx.graph) : '';
}, },
+61
View File
@@ -13,3 +13,64 @@ export function soundFileOf(project: Project, id: string): string {
const asset = findAsset(project, id); const asset = findAsset(project, id);
return asset?.type === 'sound' ? asset.file : ''; return asset?.type === 'sound' ? asset.file : '';
} }
/**
* The stretch of its file a sound asset is (#343), in seconds: `end` 0 runs to the end of the file.
* `fade` is the ramp each edge inside the file gets, so a cut through a wave does not click.
*/
export interface SoundCut {
start: number;
end: number;
fade: number;
}
/** Samples of a file a cut keeps, `from` up to `to`, and the samples of each ramp at an inner edge. */
export interface CutRange {
from: number;
to: number;
fadeIn: number;
fadeOut: number;
}
/** What a sound is played by: its file, or for a cut `<file>#<start>~<end>~<fade>`, a name of its own. */
export function cutName(file: string, cut: SoundCut): string {
if (file === '' || (cut.start <= 0 && cut.end <= 0)) return file;
return `${file}#${cut.start}~${cut.end}~${cut.fade}`;
}
/** A name `cutName` made, taken apart again; a plain file has no cut. */
export function parseCutName(name: string): { file: string; cut: SoundCut | null } {
const hash = name.indexOf('#');
if (hash < 0) return { file: name, cut: null };
const [start, end, fade] = name
.slice(hash + 1)
.split('~')
.map((part) => Number(part));
const held = (value: number | undefined) => (Number.isFinite(value) ? Math.max(0, value!) : 0);
return { file: name.slice(0, hash), cut: { start: held(start), end: held(end), fade: held(fade) } };
}
/**
* Where a cut lies in `length` samples at `rate`: at least one sample, inside the file. An edge at
* the start or the end of the file is left as it is; one inside ramps over `fade`, no more than half
* of what is kept so the two ramps never cross.
*/
export function cutRange(length: number, rate: number, cut: SoundCut): CutRange {
const last = Math.max(1, length);
const from = Math.min(last - 1, Math.max(0, Math.round(cut.start * rate)));
const asked = cut.end > 0 ? Math.min(last, Math.round(cut.end * rate)) : last;
const to = Math.max(from + 1, asked);
const ramp = Math.min(Math.round(Math.max(0, cut.fade) * rate), Math.floor((to - from) / 2));
return { from, to, fadeIn: from > 0 ? ramp : 0, fadeOut: to < last ? ramp : 0 };
}
/** The samples of each channel a cut keeps, its inner edges ramped in and out. */
export function cutChannels(channels: readonly Float32Array[], range: CutRange): Float32Array<ArrayBuffer>[] {
const { from, to, fadeIn, fadeOut } = range;
return channels.map((data) => {
const kept = data.slice(from, to);
for (let i = 0; i < fadeIn; i++) kept[i]! *= i / fadeIn;
for (let i = 0; i < fadeOut; i++) kept[kept.length - 1 - i]! *= i / fadeOut;
return kept;
});
}
+2 -1
View File
@@ -1,5 +1,6 @@
import { instrumentBuildFor, instrumentSourcesOf } from '../instrument/compile'; import { instrumentBuildFor, instrumentSourcesOf } from '../instrument/compile';
import { findAssetOf, type Project } from '../model/Project'; import { findAssetOf, type Project } from '../model/Project';
import { playedSoundById } from '../properties/playedSound';
import { soundFxBuildFor } from '../soundfx/compile'; import { soundFxBuildFor } from '../soundfx/compile';
import { DRY_VOICE, type FxPlan, type FxVoice } from '../soundfx/nodes'; import { DRY_VOICE, type FxPlan, type FxVoice } from '../soundfx/nodes';
import type { SequenceNeeds } from './playbacks'; import type { SequenceNeeds } from './playbacks';
@@ -26,7 +27,7 @@ export function sequenceNeeds(project: Project, sequence: Sequence): SequenceNee
} }
const files = new Set<string>(); const files = new Set<string>();
for (const clip of sequence.clips ?? []) { for (const clip of sequence.clips ?? []) {
const file = findAssetOf(project, clip.sound, 'sound')?.file; const file = playedSoundById(project, clip.sound);
if (file) files.add(file); if (file) files.add(file);
const fx = clipFx(project, clip.fx, 120); const fx = clipFx(project, clip.fx, 120);
if (fx !== DRY_VOICE) layers.push(fx.plan); if (fx !== DRY_VOICE) layers.push(fx.plan);
+23 -3
View File
@@ -103,7 +103,14 @@ import {
} from './particles'; } from './particles';
import { SHADER_KEY, shaderOf, shaderParamDefs } from './shaders'; import { SHADER_KEY, shaderOf, shaderParamDefs } from './shaders';
import { SHADER_RESOLUTION_KEY, SHADER_RESOLUTION_OPTIONS } from './cameraShader'; import { SHADER_RESOLUTION_KEY, SHADER_RESOLUTION_OPTIONS } from './cameraShader';
import { LOOP_KEY as SOUND_LOOP_KEY, VOLUME_KEY as SOUND_VOLUME_KEY } from './sounds'; import {
DEFAULT_FADE_MS,
END_KEY as SOUND_END_KEY,
FADE_KEY as SOUND_FADE_KEY,
LOOP_KEY as SOUND_LOOP_KEY,
START_KEY as SOUND_START_KEY,
VOLUME_KEY as SOUND_VOLUME_KEY,
} from './sounds';
import { import {
BEAT_OPTIONS, BEAT_OPTIONS,
DEFAULT_BARS, DEFAULT_BARS,
@@ -671,11 +678,24 @@ const ASSET_SCHEMAS: Record<AssetType, PropertySchema> = {
// The typeface itself is intrinsic data on the asset (see FontAsset), not a property. Nearest draws // The typeface itself is intrinsic data on the asset (see FontAsset), not a property. Nearest draws
// a pixel font a pixel per unit, magnified sharp. // a pixel font a pixel per unit, magnified sharp.
font: [filterRow(true)], font: [filterRow(true)],
// The file is intrinsic data (see SoundAsset); these two say how it is played by default, and a // The file is intrinsic data (see SoundAsset); volume and loop say how it is played by default, and
// `Play sound` node scales the one and can add to the other. // a `Play sound` node scales the one and can add to the other.
sound: [ sound: [
{ key: SOUND_VOLUME_KEY, label: 'Volume', kind: 'number', default: 1, min: 0, max: 1, step: 0.05 }, { key: SOUND_VOLUME_KEY, label: 'Volume', kind: 'number', default: 1, min: 0, max: 1, step: 0.05 },
{ key: SOUND_LOOP_KEY, label: 'Loop', kind: 'boolean', default: false }, { key: SOUND_LOOP_KEY, label: 'Loop', kind: 'boolean', default: false },
// The stretch of the file it is (#343), set in its tab; an end of 0 is the end of the file.
{ key: SOUND_START_KEY, label: 'Start (s)', kind: 'number', default: 0, min: 0, step: 0.01, decimals: 3 },
{ key: SOUND_END_KEY, label: 'End (s)', kind: 'number', default: 0, min: 0, step: 0.01, decimals: 3 },
{
key: SOUND_FADE_KEY,
label: 'Edge fade (ms)',
kind: 'number',
default: DEFAULT_FADE_MS,
min: 0,
max: 1000,
step: 1,
decimals: 0,
},
], ],
// The effects are the asset's graph (see SoundFxAsset), not a property. // The effects are the asset's graph (see SoundFxAsset), not a property.
soundfx: [], soundfx: [],
+19
View File
@@ -0,0 +1,19 @@
import type { SoundAsset } from '../model/Asset';
import { findAssetOf, type Project } from '../model/Project';
import { cutName } from '../model/sound';
import { resolveAssetProps } from './resolve';
import { cutOf } from './sounds';
/**
* What a sound asset is played by (#343): its file, or the stretch of it the asset is cut to. Every
* way a sound is heard asks this rather than for the file – the cut is the sample.
*/
export function playedSoundOf(project: Project, asset: SoundAsset): string {
return cutName(asset.file, cutOf(resolveAssetProps(project, asset)));
}
/** The same by id, empty when the id is unset or no longer names a sound. */
export function playedSoundById(project: Project, id: string): string {
const asset = id === '' ? undefined : findAssetOf(project, id, 'sound');
return asset ? playedSoundOf(project, asset) : '';
}
+17
View File
@@ -1,10 +1,17 @@
import { booleanOf, type PropertyBag } from '../model/Properties'; import { booleanOf, type PropertyBag } from '../model/Properties';
import type { SoundCut } from '../model/sound';
import { coerceAs } from './kinds'; import { coerceAs } from './kinds';
/** How loud this sound is by itself, 0–1; a `Play sound` node scales this rather than replacing it. */ /** How loud this sound is by itself, 0–1; a `Play sound` node scales this rather than replacing it. */
export const VOLUME_KEY = 'volume'; export const VOLUME_KEY = 'volume';
/** Whether this sound is music: it repeats until something stops it. */ /** Whether this sound is music: it repeats until something stops it. */
export const LOOP_KEY = 'loop'; export const LOOP_KEY = 'loop';
/** Where in its file the sound starts and ends, in seconds; an end of 0 is the end of the file (#343). */
export const START_KEY = 'start';
export const END_KEY = 'end';
/** Milliseconds each cut edge inside the file ramps over, so it does not click. */
export const FADE_KEY = 'fade';
export const DEFAULT_FADE_MS = 10;
export function volumeOf(bag: PropertyBag): number { export function volumeOf(bag: PropertyBag): number {
return coerceAs(bag[VOLUME_KEY], 'number', 1); return coerceAs(bag[VOLUME_KEY], 'number', 1);
@@ -13,3 +20,13 @@ export function volumeOf(bag: PropertyBag): number {
export function loopsOf(bag: PropertyBag): boolean { export function loopsOf(bag: PropertyBag): boolean {
return booleanOf(bag[LOOP_KEY]); return booleanOf(bag[LOOP_KEY]);
} }
/** The stretch of its file a sound is, its fade in seconds. */
export function cutOf(bag: PropertyBag): SoundCut {
const seconds = (key: string, fallback: number) => Math.max(0, coerceAs(bag[key], 'number', fallback));
return {
start: seconds(START_KEY, 0),
end: seconds(END_KEY, 0),
fade: seconds(FADE_KEY, DEFAULT_FADE_MS) / 1000,
};
}
+10 -1
View File
@@ -84,6 +84,7 @@ import { copySelection } from './clipboard';
import { PixelArtEditor } from './pixelart/PixelArtEditor'; import { PixelArtEditor } from './pixelart/PixelArtEditor';
import { VectorArtEditor } from './vectorart/VectorArtEditor'; import { VectorArtEditor } from './vectorart/VectorArtEditor';
import { CurveEditor } from './curve/CurveEditor'; import { CurveEditor } from './curve/CurveEditor';
import { SoundEditor } from './sound/SoundEditor';
import { TextureEditor } from './texture/TextureEditor'; import { TextureEditor } from './texture/TextureEditor';
import { installFileImport } from './fileImport'; import { installFileImport } from './fileImport';
import type { PluginManager } from './plugins/loader'; import type { PluginManager } from './plugins/loader';
@@ -97,6 +98,7 @@ export type CentreTab =
| 'statemachine' | 'statemachine'
| 'shader' | 'shader'
| 'particles' | 'particles'
| 'sound'
| 'soundfx' | 'soundfx'
| 'instrument' | 'instrument'
| 'pattern' | 'pattern'
@@ -150,6 +152,7 @@ const PANE_ORDER = [
'statemachine', 'statemachine',
'shader', 'shader',
'particles', 'particles',
'sound',
'soundfx', 'soundfx',
'instrument', 'instrument',
'pattern', 'pattern',
@@ -173,6 +176,7 @@ const TAB_LABELS: Record<CentreTab, string> = {
statemachine: 'State machine', statemachine: 'State machine',
shader: 'Shader', shader: 'Shader',
particles: 'Particles', particles: 'Particles',
sound: 'Sound',
soundfx: 'Sound FX', soundfx: 'Sound FX',
instrument: 'Instrument', instrument: 'Instrument',
pattern: 'Pattern', pattern: 'Pattern',
@@ -225,6 +229,7 @@ export class Editor implements PlayHost {
readonly stateMachineEditor: StateMachineEditor; readonly stateMachineEditor: StateMachineEditor;
readonly shaderEditor: ShaderEditor; readonly shaderEditor: ShaderEditor;
readonly particlesEditor: ParticlesEditor; readonly particlesEditor: ParticlesEditor;
readonly soundEditor: SoundEditor;
readonly soundFxEditor: SoundFxEditor; readonly soundFxEditor: SoundFxEditor;
readonly instrumentEditor: InstrumentEditor; readonly instrumentEditor: InstrumentEditor;
readonly patternEditor: PatternEditor; readonly patternEditor: PatternEditor;
@@ -314,6 +319,7 @@ export class Editor implements PlayHost {
this.stateMachineEditor = new StateMachineEditor(state); this.stateMachineEditor = new StateMachineEditor(state);
this.shaderEditor = new ShaderEditor(state); this.shaderEditor = new ShaderEditor(state);
this.particlesEditor = new ParticlesEditor(state); this.particlesEditor = new ParticlesEditor(state);
this.soundEditor = new SoundEditor(state);
this.soundFxEditor = new SoundFxEditor(state); this.soundFxEditor = new SoundFxEditor(state);
this.instrumentEditor = new InstrumentEditor(state); this.instrumentEditor = new InstrumentEditor(state);
this.patternEditor = new PatternEditor(state); this.patternEditor = new PatternEditor(state);
@@ -426,6 +432,7 @@ export class Editor implements PlayHost {
statemachine: this.stateMachineEditor, statemachine: this.stateMachineEditor,
shader: this.shaderEditor, shader: this.shaderEditor,
particles: this.particlesEditor, particles: this.particlesEditor,
sound: this.soundEditor,
soundfx: this.soundFxEditor, soundfx: this.soundFxEditor,
instrument: this.instrumentEditor, instrument: this.instrumentEditor,
pattern: this.patternEditor, pattern: this.patternEditor,
@@ -1023,6 +1030,7 @@ export class Editor implements PlayHost {
if (this.tab === 'curve') this.curveEditor.tick(dt); if (this.tab === 'curve') this.curveEditor.tick(dt);
if (this.tab === 'pattern') this.patternEditor.tick(); if (this.tab === 'pattern') this.patternEditor.tick();
if (this.tab === 'song') this.songEditor.tick(); if (this.tab === 'song') this.songEditor.tick();
if (this.tab === 'sound') this.soundEditor.tick();
// Play repaints the machine tab itself (see `PlaySession.tick`); an editor run has no session // Play repaints the machine tab itself (see `PlaySession.tick`); an editor run has no session
// to do it, and its states move just the same. // to do it, and its states move just the same.
if (this.tab === 'statemachine' && this.editorRun.isRunning) this.stateMachineEditor.canvas.redraw(); if (this.tab === 'statemachine' && this.editorRun.isRunning) this.stateMachineEditor.canvas.redraw();
@@ -1057,7 +1065,8 @@ export class Editor implements PlayHost {
(this.tab === 'tile' && this.tileEditor.isPlaying) || (this.tab === 'tile' && this.tileEditor.isPlaying) ||
(this.tab === 'curve' && this.curveEditor.isPlaying) || (this.tab === 'curve' && this.curveEditor.isPlaying) ||
(this.tab === 'pattern' && this.patternEditor.isPlaying) || (this.tab === 'pattern' && this.patternEditor.isPlaying) ||
(this.tab === 'song' && this.songEditor.isPlaying) (this.tab === 'song' && this.songEditor.isPlaying) ||
(this.tab === 'sound' && this.soundEditor.isPlaying)
); );
} }
+1
View File
@@ -9,6 +9,7 @@ export const TAB_OF_ASSET: Partial<Record<AssetType, CentreTab>> = {
shader: 'shader', shader: 'shader',
shaderfunction: 'shader', shaderfunction: 'shader',
particles: 'particles', particles: 'particles',
sound: 'sound',
soundfx: 'soundfx', soundfx: 'soundfx',
instrument: 'instrument', instrument: 'instrument',
pattern: 'pattern', pattern: 'pattern',
+44
View File
@@ -0,0 +1,44 @@
import { findAssetOf } from '../../core/model/Project';
import { playedSoundOf } from '../../core/properties/playedSound';
import { soundLibrary } from '../../viewport/SoundLibrary';
import { centreOf, type HookContext } from './hookHelpers';
/** The Sound tab (#343). */
export function soundHooks({ state, editor }: HookContext) {
const { soundEditor } = editor;
return {
soundTab: {
current: () => soundEditor.currentAssetId,
info: () => soundEditor.shownInfo,
rect: () => soundEditor.canvasRect(),
/** Where a second of the file is on screen, at the middle of the wave. */
at: (seconds: number) => soundEditor.canvas.secondsToScreen(seconds),
/** The stretch between the handles, in seconds; null while the file is not there. */
span: () => soundEditor.canvas.shownSpan(),
duration: () => soundEditor.canvas.fileDuration(),
cursor: () => soundEditor.canvas.cursor,
playhead: () => soundEditor.canvas.playhead,
playing: () => soundEditor.isPlaying,
playButton: () => ({ ...centreOf(soundEditor.playButton), text: soundEditor.playButton.text }),
wholeButton: () => ({ ...centreOf(soundEditor.wholeButton), enabled: soundEditor.wholeButton.enabled }),
fadeField: () => ({ ...centreOf(soundEditor.fadeField), text: soundEditor.fadeField.value }),
/** What a sound asset is played by: its file, or the name of its cut. */
played: (assetId: string) => {
const asset = findAssetOf(state.project, assetId, 'sound');
return asset ? playedSoundOf(state.project, asset) : null;
},
/** The samples a name plays, cut and faded: their count, rate, and the first and last few of channel 0. */
samples: async (name: string) => {
const buffer = await soundLibrary.load(name);
if (!buffer) return null;
const data = buffer.getChannelData(0);
return {
length: buffer.length,
rate: buffer.sampleRate,
head: Array.from(data.slice(0, 4)),
tail: Array.from(data.slice(-4)),
};
},
},
};
}
+2
View File
@@ -13,6 +13,7 @@ import {
stageNodes, stageNodes,
} from '../viewport/SoundLibrary'; } from '../viewport/SoundLibrary';
import { curveHooks } from './debug/curveHooks'; import { curveHooks } from './debug/curveHooks';
import { soundHooks } from './debug/soundHooks';
import { patternHooks } from './debug/patternHooks'; import { patternHooks } from './debug/patternHooks';
import { songHooks } from './debug/songHooks'; import { songHooks } from './debug/songHooks';
import { documentHooks } from './debug/documentHooks'; import { documentHooks } from './debug/documentHooks';
@@ -48,6 +49,7 @@ export function installDebugHook(app: Application, state: EditorState, editor: E
...curveHooks(context), ...curveHooks(context),
...patternHooks(context), ...patternHooks(context),
...songHooks(context), ...songHooks(context),
...soundHooks(context),
...renderHooks(context), ...renderHooks(context),
/** /**
* Voices started, stopped and in flight. CI runs the browser with `--mute-audio`, so nothing * Voices started, stopped and in flight. CI runs the browser with `--mute-audio`, so nothing
+2 -2
View File
@@ -167,10 +167,10 @@ export function buildAssetSection(host: InspectorHost, assetId: string): void {
host.body.add( host.body.add(
asset.file ? new FontPreview(asset.file) : new Label('Not uploaded.', { style: 'warning' }), asset.file ? new FontPreview(asset.file) : new Label('Not uploaded.', { style: 'warning' }),
); );
// A sound has two rows above this; the preview is the only way to find out which sound it is. // A sound has its rows above this; the preview is the quickest way to find out which sound it is.
if (asset.type === 'sound') if (asset.type === 'sound')
host.body.add( host.body.add(
asset.file ? new SoundPreview(asset.file) : new Label('Not uploaded.', { style: 'warning' }), asset.file ? new SoundPreview(host.state, assetId) : new Label('Not uploaded.', { style: 'warning' }),
); );
if (asset.type === 'sprite') addSpritePreview(host, { kind: 'asset', assetId }); if (asset.type === 'sprite') addSpritePreview(host, { kind: 'asset', assetId });
} }
+13 -11
View File
@@ -42,6 +42,7 @@ import {
Widget, Widget,
zoomFactor, zoomFactor,
} from '../../ui'; } from '../../ui';
import { playedSoundById, playedSoundOf } from '../../core/properties/playedSound';
import { soundLibrary } from '../../viewport/SoundLibrary'; import { soundLibrary } from '../../viewport/SoundLibrary';
import { TimeRuler } from '../music/TimeRuler'; import { TimeRuler } from '../music/TimeRuler';
import { import {
@@ -940,8 +941,9 @@ export class SongTimeline extends Widget {
if (dragged.type === 'pattern') if (dragged.type === 'pattern')
length = droppedLength(null, patternLength(dragged), songTempo(asset.props), bar); length = droppedLength(null, patternLength(dragged), songTempo(asset.props), bar);
else { else {
this.ensureLoaded(dragged.file); const played = playedSoundOf(this.state.project, dragged);
length = droppedLength(soundLibrary.duration(dragged.file), null, songTempo(asset.props), bar); this.ensureLoaded(played);
length = droppedLength(soundLibrary.duration(played), null, songTempo(asset.props), bar);
} }
this.drop = { kind: dragged.type, asset: dragged.id, track: asset.tracks[row]!.id, tick, length }; this.drop = { kind: dragged.type, asset: dragged.id, track: asset.tracks[row]!.id, tick, length };
this.invalidate(); this.invalidate();
@@ -1092,8 +1094,8 @@ export class SongTimeline extends Widget {
const pattern = findAssetOf(this.state.project, item.asset, 'pattern'); const pattern = findAssetOf(this.state.project, item.asset, 'pattern');
return pattern ? patternLength(pattern) : null; return pattern ? patternLength(pattern) : null;
} }
const sound = findAssetOf(this.state.project, item.asset, 'sound'); const played = playedSoundById(this.state.project, item.asset);
const seconds = sound?.file ? soundLibrary.duration(sound.file) : null; const seconds = played ? soundLibrary.duration(played) : null;
return seconds ? ticksOfSeconds(seconds, tempo) : null; return seconds ? ticksOfSeconds(seconds, tempo) : null;
} }
@@ -1148,8 +1150,8 @@ export class SongTimeline extends Widget {
const pattern = findAssetOf(this.state.project, item.asset, 'pattern'); const pattern = findAssetOf(this.state.project, item.asset, 'pattern');
return pattern && pattern.notes.length > 0 ? 'notes' : 'none'; return pattern && pattern.notes.length > 0 ? 'notes' : 'none';
} }
const sound = findAssetOf(this.state.project, item.asset, 'sound'); const played = playedSoundById(this.state.project, item.asset);
return sound?.file && soundLibrary.peaks(sound.file) ? 'wave' : 'none'; return played && soundLibrary.peaks(played) ? 'wave' : 'none';
} }
private ensureLoaded(file: string): void { private ensureLoaded(file: string): void {
@@ -1397,12 +1399,12 @@ export class SongTimeline extends Widget {
* round is drawn from its own start in the same columns, so every repeat looks alike. * round is drawn from its own start in the same columns, so every repeat looks alike.
*/ */
private drawWave(item: SongItem, box: Box, lw: number): void { private drawWave(item: SongItem, box: Box, lw: number): void {
const sound = findAssetOf(this.state.project, item.asset, 'sound'); const played = playedSoundById(this.state.project, item.asset);
if (!sound?.file) return; if (!played) return;
const peaks = soundLibrary.peaks(sound.file); const peaks = soundLibrary.peaks(played);
const seconds = soundLibrary.duration(sound.file); const seconds = soundLibrary.duration(played);
if (!peaks || !seconds) { if (!peaks || !seconds) {
this.ensureLoaded(sound.file); this.ensureLoaded(played);
return; return;
} }
const asset = this.asset()!; const asset = this.asset()!;
+149
View File
@@ -0,0 +1,149 @@
import { SetAssetPropertyCommand } from '../../core/commands/assetCommands';
import type { Asset, SoundAsset } from '../../core/model/Asset';
import { playedSoundOf } from '../../core/properties/playedSound';
import { resolveAssetProps } from '../../core/properties/resolve';
import { cutOf, FADE_KEY, volumeOf } from '../../core/properties/sounds';
import { Button, Label, NumberField } from '../../ui';
import { soundLibrary } from '../../viewport/SoundLibrary';
import { AssetTabEditor } from '../AssetTabEditor';
import { claimTransport, releaseTransport } from '../music/transport';
import { TOOLBAR_ICON_SVG } from '../panels/toolbarIcons';
import type { EditorState } from '../state';
import { WaveCanvas } from './WaveCanvas';
const INFO_WIDTH = 140;
const FADE_WIDTH = 56;
const PREVIEW_OWNER = 'sound-preview';
/**
* The Sound tab (#343): the file as a waveform, and the stretch of it the asset is. That stretch is
* the sample – a song item, a `Play sound` and an instrument's Sample all play it and nothing else.
* The header plays it (Space does too), sets the fade at its edges and puts the whole file back.
*/
export class SoundEditor extends AssetTabEditor<SoundAsset, WaveCanvas> {
/** Public for the check scripts. */
readonly playButton: Button;
readonly wholeButton: Button;
readonly fadeField: NumberField;
/** The voice the tab plays, 0 while none, and the sound it is of. */
private voice = 0;
private playingId = '';
private readonly halt = () => this.stop();
constructor(state: EditorState) {
super(state, { infoWidth: INFO_WIDTH });
this.playButton = new Button('Play', {
icon: TOOLBAR_ICON_SVG.play,
minWidth: 72,
tooltip: 'Play the stretch between the handles (Space)',
onClick: () => this.togglePlay(),
});
this.wholeButton = new Button('Whole file', {
tooltip: 'Play the whole file again: the handles go back to its start and end',
onClick: () => this.canvas.resetCut(),
});
this.fadeField = new NumberField(0, {
width: FADE_WIDTH,
min: 0,
max: 1000,
step: 1,
decimals: 0,
onChange: (ms) => this.setFade(ms),
});
const canvas = new WaveCanvas(state);
canvas.onTogglePlay = () => this.togglePlay();
canvas.onLoaded = () => this.refresh();
this.mount(canvas, [
this.playButton,
this.wholeButton,
new Label('Edge fade (ms)', { style: 'muted' }),
this.fadeField,
]);
state.on('load', () => this.stop());
}
override selectedId(): string {
return this.state.selectedSoundId ?? '';
}
override isMine(asset: Asset): asset is SoundAsset {
return asset.type === 'sound';
}
get isPlaying(): boolean {
return this.voice !== 0;
}
togglePlay(): void {
if (this.isPlaying) this.stop();
else this.play();
}
/** Plays the stretch once, at the sound's own volume, from its start. */
play(): void {
const asset = this.asset();
if (!asset?.file) return;
this.stop();
claimTransport(this.halt);
const volume = volumeOf(resolveAssetProps(this.state.project, asset));
this.voice = soundLibrary.play(playedSoundOf(this.state.project, asset), volume, false, PREVIEW_OWNER);
this.playingId = this.voice ? asset.id : '';
this.syncPlayButton();
}
stop(): void {
releaseTransport(this.halt);
if (this.voice) soundLibrary.stopOwner(PREVIEW_OWNER);
this.voice = 0;
this.playingId = '';
this.canvas.playhead = null;
this.canvas.invalidate();
this.syncPlayButton();
}
/** Only called while the tab is visible: the playhead follows what is heard, and stops with it. */
tick(): void {
if (!this.voice) return;
if (!soundLibrary.playing(this.voice)) return this.stop();
const started = soundLibrary.startedAt(this.voice);
const span = this.canvas.shownSpan();
if (started === null || !span) return;
const heard = soundLibrary.clock() - started - soundLibrary.latency();
this.canvas.playhead = span.start + Math.max(0, Math.min(span.end - span.start, heard));
this.canvas.invalidate();
}
private setFade(ms: number): void {
const asset = this.asset();
const now = asset && Math.round(cutOf(resolveAssetProps(this.state.project, asset)).fade * 1000);
if (!asset || ms === now) return;
this.state.execute(new SetAssetPropertyCommand(asset.id, FADE_KEY, ms));
}
private syncPlayButton(): void {
this.playButton.setIcon(this.isPlaying ? TOOLBAR_ICON_SVG.stop : TOOLBAR_ICON_SVG.play);
this.playButton.setText(this.isPlaying ? 'Stop' : 'Play');
this.playButton.setEnabled(Boolean(this.asset()?.file));
}
protected override infoText(asset: SoundAsset): string {
const span = this.canvas.shownSpan();
const whole = this.canvas.fileDuration();
if (!asset.file) return 'Not uploaded';
if (!span || whole === null) return '';
const kept = span.end - span.start;
return kept >= whole ? `${whole.toFixed(2)} s` : `${kept.toFixed(2)} s of ${whole.toFixed(2)} s`;
}
protected override syncExtras(asset: SoundAsset | null): void {
const ready = asset !== null && Boolean(asset.file);
this.fitButton.setEnabled(ready);
this.wholeButton.setEnabled(ready);
this.fadeField.setEnabled(ready);
const fade = asset ? cutOf(resolveAssetProps(this.state.project, asset)).fade : 0;
this.fadeField.setNumber(Math.round(fade * 1000));
// Another sound on screen: what was playing is no longer what the tab shows.
if (this.isPlaying && asset?.id !== this.playingId) this.stop();
this.syncPlayButton();
}
}
+28 -9
View File
@@ -1,29 +1,40 @@
import { playedSoundById } from '../../core/properties/playedSound';
import { Button, HStack, Label, Theme } from '../../ui'; import { Button, HStack, Label, Theme } from '../../ui';
import { soundLibrary } from '../../viewport/SoundLibrary'; import { soundLibrary } from '../../viewport/SoundLibrary';
import { TOOLBAR_ICON_SVG } from '../panels/toolbarIcons'; import { TOOLBAR_ICON_SVG } from '../panels/toolbarIcons';
import type { EditorState } from '../state';
/** /**
* What a sound asset looks like: a button that plays it and the length the decoder reports. There * What a sound asset sounds like: a button that plays it and the length the decoder reports. Both
* is nothing else to show – its file name is a hash, and the waveform would be a picture of bytes * are of the stretch the asset is cut to (#343); the waveform is in its tab.
* nobody reads.
* *
* Playing here goes through the same library a running game uses, so what the inspector plays is * Playing here goes through the same library a running game uses, so what the inspector plays is
* exactly what the graph will: same decode, same device, and `Stop` reaches both. * exactly what the graph will: same decode, same device, and `Stop` reaches both.
*/ */
export class SoundPreview extends HStack { export class SoundPreview extends HStack {
private readonly length: Label; private readonly length: Label;
private shown = '';
private readonly stopWatching: () => void;
constructor(private readonly file: string) { constructor(
private readonly state: EditorState,
private readonly assetId: string,
) {
super({ gap: Theme.gap, align: 'center' }); super({ gap: Theme.gap, align: 'center' });
const play = new Button('Play', { icon: TOOLBAR_ICON_SVG.play }); const play = new Button('Play', { icon: TOOLBAR_ICON_SVG.play });
play.onClick = () => soundLibrary.play(this.file, 1, false); play.onClick = () => soundLibrary.play(this.played(), 1, false);
this.add(play); this.add(play);
const stop = new Button('Stop', { icon: TOOLBAR_ICON_SVG.stop }); const stop = new Button('Stop', { icon: TOOLBAR_ICON_SVG.stop });
stop.onClick = () => soundLibrary.stop(this.file); stop.onClick = () => soundLibrary.stop(this.played());
this.add(stop); this.add(stop);
this.length = new Label('', { style: 'muted' }); this.length = new Label('', { style: 'muted' });
this.add(this.length, 1); this.add(this.length, 1);
this.showLength(); this.showLength();
this.stopWatching = state.on('project', () => this.showLength());
}
private played(): string {
return playedSoundById(this.state.project, this.assetId);
} }
/** /**
@@ -32,14 +43,22 @@ export class SoundPreview extends HStack {
* which is more honest than a number guessed from the file size. * which is more honest than a number guessed from the file size.
*/ */
private showLength(): void { private showLength(): void {
const known = soundLibrary.duration(this.file); const name = this.played();
if (name === this.shown) return;
this.shown = name;
const known = soundLibrary.duration(name);
if (known !== null) { if (known !== null) {
this.length.setText(`${known.toFixed(2)} s`); this.length.setText(`${known.toFixed(2)} s`);
return; return;
} }
void soundLibrary.load(this.file).then((buffer) => { void soundLibrary.load(name).then((buffer) => {
if (this.destroyed) return; if (this.destroyed || this.shown !== name) return;
this.length.setText(buffer ? `${buffer.duration.toFixed(2)} s` : 'Could not be decoded.'); this.length.setText(buffer ? `${buffer.duration.toFixed(2)} s` : 'Could not be decoded.');
}); });
} }
override destroy(options?: Parameters<HStack['destroy']>[0]): void {
this.stopWatching();
super.destroy(options);
}
} }
+458
View File
@@ -0,0 +1,458 @@
import {
Container,
type FederatedPointerEvent,
type FederatedWheelEvent,
Graphics,
type Text,
} from 'pixi.js';
import { CompoundCommand } from '../../core/commands/CompoundCommand';
import { SetAssetPropertyCommand } from '../../core/commands/assetCommands';
import type { SoundAsset } from '../../core/model/Asset';
import type { Vec2 } from '../../core/model/Properties';
import { findAssetOf } from '../../core/model/Project';
import { cutRange } from '../../core/model/sound';
import { resolveAssetProps } from '../../core/properties/resolve';
import { cutOf, END_KEY, START_KEY } from '../../core/properties/sounds';
import { crispText, type Size, textStyle, Theme, ui, Widget, zoomFactor } from '../../ui';
import { soundLibrary } from '../../viewport/SoundLibrary';
import type { EditorState } from '../state';
import {
clampWave,
type CutSpan,
draggedSpan,
edgeAt,
fitWave,
rulerLabel,
rulerStep,
secondsToX,
spanOf,
spanPeak,
storedCut,
type WavePeaks,
wavePeaks,
type WaveView,
xToSeconds,
zoomWave,
} from './waveView';
const RULER_HEIGHT = 24;
/** Room either side of the file when it is fitted. */
const FIT_MARGIN = 16;
/** Pixels apart the ruler's labels are at least. */
const LABEL_SPACING = 70;
/** The shortest span a drag leaves. */
const LEAST_SPAN = 0.001;
/** Pixels a press has to travel before it spans a new stretch. */
const DRAG_START = 3;
type Gesture =
| { kind: 'pan'; pointerX: number; panX: number }
| {
kind: 'span';
edge: 'start' | 'end' | null;
from: number;
pointerX: number;
moved: boolean;
span: CutSpan;
};
/** A file's samples as the canvas draws them: decoded once, its block peaks made on first sight. */
interface Wave {
duration: number;
rate: number;
channels: Float32Array[];
peaks: WavePeaks;
}
const waves = new WeakMap<AudioBuffer, Wave>();
function waveOf(buffer: AudioBuffer): Wave {
const known = waves.get(buffer);
if (known) return known;
const channels = Array.from({ length: buffer.numberOfChannels }, (_, c) => buffer.getChannelData(c));
const wave = { duration: buffer.duration, rate: buffer.sampleRate, channels, peaks: wavePeaks(channels) };
waves.set(buffer, wave);
return wave;
}
/**
* The Sound tab's canvas (#343): the whole file as one waveform, the loudest of its channels, under
* a ruler in seconds. The stretch the asset is cut to stands between two handles, what lies outside
* it darker; dragging a handle moves that edge, dragging beside them spans a new stretch. One drag
* is one undo step. The wheel zooms, the middle button pans, as in the music tabs.
*/
export class WaveCanvas extends Widget {
private readonly bg = new Graphics();
private readonly waveG = new Graphics();
private readonly overlay = new Graphics();
private readonly labels = new Container();
/** Wave, cut and ruler, kept to the widget: a handle panned out of view must not draw beside it. */
private readonly content = new Container();
private readonly clip = new Graphics();
private readonly texts: Text[] = [];
private readonly hint = crispText({ text: '', style: textStyle('muted') });
private view: WaveView = { panX: FIT_MARGIN, pxPerSecond: 100 };
private needsFit = true;
private assetId = '';
private gesture: Gesture | null = null;
/** The files asked for, so a wave is drawn once its bytes are there. */
private readonly loading = new Set<string>();
/** Where the tab's playing stands in the file, in seconds; null while it does not play. */
playhead: number | null = null;
onTogglePlay: () => void = () => {};
/** The file's samples arrived: the tab's readout can say how long it is. */
onLoaded: () => void = () => {};
constructor(private readonly state: EditorState) {
super();
this.focusable = true;
this.labels.eventMode = 'none';
this.hint.eventMode = 'none';
this.content.addChild(this.waveG, this.overlay, this.labels);
this.content.mask = this.clip;
this.addChild(this.bg, this.content, this.clip, this.hint);
this.on('pointerdown', (e: FederatedPointerEvent) => this.onPointerDown(e));
this.on('globalpointermove', (e: FederatedPointerEvent) => this.onPointerMove(e));
this.onPointerRelease(() => this.onPointerUp());
this.on('wheel', (e: FederatedWheelEvent) => this.onWheel(e));
state.on('project', () => this.invalidate());
}
override measure(): Size {
return { w: Math.max(this.minWidth, 360), h: Math.max(this.minHeight, 160) };
}
get currentAssetId(): string {
return this.assetId;
}
show(assetId: string): void {
if (this.assetId === assetId) return;
this.gesture = null;
this.assetId = assetId;
this.needsFit = true;
this.redraw();
}
saveView(): WaveView {
return { ...this.view };
}
restoreView(view: unknown): void {
const held = view as Partial<WaveView>;
if (typeof held.panX !== 'number' || typeof held.pxPerSecond !== 'number') return;
this.view = { panX: held.panX, pxPerSecond: held.pxPerSecond };
this.needsFit = false;
this.redraw();
}
/** The whole file across. */
fit(): void {
const wave = this.wave();
if (!wave || this.w <= 0) {
this.needsFit = true;
return;
}
this.view = fitWave(wave.duration, this.w, FIT_MARGIN);
this.needsFit = false;
this.redraw();
}
/** The stretch the asset is cut to, as shown – while a drag runs, as it would leave it. */
shownSpan(): CutSpan | null {
const asset = this.asset();
const wave = this.wave();
if (!asset || !wave) return null;
if (this.gesture?.kind === 'span') return this.gesture.span;
const cut = cutOf(resolveAssetProps(this.state.project, asset));
return spanOf(cut.start, cut.end, wave.duration);
}
/** The file's length in seconds, null while it is not loaded (test hook). */
fileDuration(): number | null {
return this.wave()?.duration ?? null;
}
/** Where a second of the file is on screen, at the middle of the wave (test hook). */
secondsToScreen(seconds: number): Vec2 {
const origin = this.getGlobalPosition();
return { x: origin.x + secondsToX(this.view, seconds), y: origin.y + RULER_HEIGHT + this.waveHeight / 2 };
}
/** Puts the whole file back, in one undo step. */
resetCut(): void {
const asset = this.asset();
const wave = this.wave();
if (asset && wave) this.commit(asset, { start: 0, end: wave.duration }, wave, 'Reset sound cut');
}
private asset(): SoundAsset | null {
return (this.assetId && findAssetOf(this.state.project, this.assetId, 'sound')) || null;
}
/** The file's samples, asked for the first time they are not there. */
private wave(): Wave | null {
const file = this.asset()?.file;
if (!file) return null;
const buffer = soundLibrary.decoded(file);
if (buffer) return waveOf(buffer);
if (!this.loading.has(file)) {
this.loading.add(file);
void soundLibrary.load(file).then(() => {
this.invalidate();
this.onLoaded();
});
}
return null;
}
private get waveHeight(): number {
return Math.max(0, this.h - RULER_HEIGHT);
}
override onKeyDown(event: KeyboardEvent): boolean {
if (event.code !== 'Space' || event.ctrlKey || event.metaKey || !this.asset()) return false;
this.onTogglePlay();
return true;
}
private onPointerDown(e: FederatedPointerEvent): void {
const wave = this.wave();
const span = this.shownSpan();
if (!wave || !span) return;
const p = this.toLocal(e.global);
e.stopPropagation();
ui().focus.focus(this);
if (e.button === 1) {
this.gesture = { kind: 'pan', pointerX: p.x, panX: this.view.panX };
return;
}
if (e.button !== 0) return;
const from = Math.max(0, Math.min(wave.duration, xToSeconds(this.view, p.x)));
this.gesture = {
kind: 'span',
edge: edgeAt(this.view, span, p.x),
from,
pointerX: p.x,
moved: false,
span,
};
}
private onPointerMove(e: FederatedPointerEvent): void {
const p = this.toLocal(e.global);
const g = this.gesture;
const wave = this.wave();
if (!g || !wave) {
this.updateCursor(p.x, p.y);
return;
}
if (g.kind === 'pan') {
this.view = clampWave({ ...this.view, panX: g.panX + p.x - g.pointerX }, wave.duration, this.w);
this.needsFit = false;
this.redraw();
return;
}
if (!g.moved && g.edge === null && Math.abs(p.x - g.pointerX) < DRAG_START) return;
g.moved = true;
const before = this.committedSpan(wave);
if (!before) return;
const at = xToSeconds(this.view, p.x);
g.span = draggedSpan(before, g.edge, g.from, at, wave.duration, LEAST_SPAN);
this.redraw();
}
private onPointerUp(): void {
const g = this.gesture;
this.gesture = null;
if (g?.kind !== 'span' || !g.moved) return this.redraw();
const asset = this.asset();
const wave = this.wave();
if (asset && wave) this.commit(asset, g.span, wave, g.edge ? 'Move sound cut' : 'Cut sound');
this.redraw();
}
private committedSpan(wave: Wave): CutSpan | null {
const asset = this.asset();
if (!asset) return null;
const cut = cutOf(resolveAssetProps(this.state.project, asset));
return spanOf(cut.start, cut.end, wave.duration);
}
/** Writes a span as the asset's start and end, both in one step; nothing where neither changes. */
private commit(asset: SoundAsset, span: CutSpan, wave: Wave, label: string): void {
const next = storedCut(span, wave.duration, wave.rate);
const now = cutOf(resolveAssetProps(this.state.project, asset));
const commands = [];
if (next.start !== now.start) commands.push(new SetAssetPropertyCommand(asset.id, START_KEY, next.start));
if (next.end !== now.end) commands.push(new SetAssetPropertyCommand(asset.id, END_KEY, next.end));
if (commands.length > 0) this.state.execute(new CompoundCommand(label, commands));
}
private onWheel(e: FederatedWheelEvent): void {
const wave = this.wave();
if (!wave) return;
const p = this.toLocal(e.global);
this.view = clampWave(zoomWave(this.view, zoomFactor(e.deltaY), p.x), wave.duration, this.w);
this.needsFit = false;
this.redraw();
e.preventDefault();
}
private updateCursor(x: number, y: number): void {
const span = this.shownSpan();
const inside = x >= 0 && x <= this.w && y >= 0 && y <= this.h;
const cursor = inside && span && edgeAt(this.view, span, x) ? 'ew-resize' : 'default';
if (this.cursor !== cursor) this.cursor = cursor;
}
override layout(): void {
if (this.needsFit) this.fit();
}
override redraw(): void {
const wave = this.wave();
if (this.needsFit && wave && this.w > 0) return this.fit();
const c = Theme.colors;
const { w, h } = this;
const wh = this.waveHeight;
this.bg.clear().rect(0, 0, w, h).fill(c.viewport);
this.bg.rect(0, 0, w, RULER_HEIGHT).fill(c.panelHeader);
this.bg.rect(0, RULER_HEIGHT - 1, w, 1).fill(c.border);
this.clip.clear().rect(0, 0, w, h).fill(0xffffff);
this.waveG.clear();
this.overlay.clear();
const asset = this.asset();
const span = this.shownSpan();
this.hint.visible = !wave;
this.hint.text = !asset
? 'No sound selected – pick one in the Assets panel.'
: !asset.file
? 'Not uploaded.'
: soundLibrary.failedToLoad(asset.file)
? 'Could not be decoded.'
: 'Loading…';
this.hint.position.set(Theme.padding * 2, RULER_HEIGHT + Theme.padding * 2);
if (!wave || !span || !asset) {
this.syncLabels([]);
return;
}
const x0 = secondsToX(this.view, 0);
const x1 = secondsToX(this.view, wave.duration);
const mid = RULER_HEIGHT + wh / 2;
const half = Math.max(1, wh / 2 - 6);
// The file itself a shade lighter than what lies before and after it.
const left = Math.max(0, x0);
const right = Math.min(w, x1);
if (right > left) this.bg.rect(left, RULER_HEIGHT, right - left, wh).fill({ color: c.panel, alpha: 0.5 });
if (right > left) this.waveG.rect(left, mid, right - left, 1).fill({ color: c.border });
this.drawWave(wave, left, right, mid, half);
this.drawRuler(left, right);
this.drawCut(asset, wave, span, mid, half);
if (this.playhead !== null) {
const x = secondsToX(this.view, this.playhead);
if (x >= 0 && x <= w) this.overlay.rect(x - 0.5, 0, 1, h).fill(c.hot);
}
}
/** A bar from the lowest to the highest sample per pixel, or the samples themselves zoomed far in. */
private drawWave(wave: Wave, left: number, right: number, mid: number, half: number): void {
const color = { color: 0xffffff, alpha: 0.75 };
const perPx = wave.rate / this.view.pxPerSecond;
if (perPx < 1) {
const first = Math.max(0, Math.floor(xToSeconds(this.view, left) * wave.rate));
const last = Math.min(
wave.channels[0]!.length - 1,
Math.ceil(xToSeconds(this.view, right) * wave.rate),
);
const points: number[] = [];
for (let i = first; i <= last; i++) {
const v = wave.channels.reduce(
(most, data) => (Math.abs(data[i]!) > Math.abs(most) ? data[i]! : most),
0,
);
points.push(secondsToX(this.view, i / wave.rate), mid - Math.max(-1, Math.min(1, v)) * half);
}
if (points.length >= 4) this.waveG.poly(points, false).stroke({ width: 1, ...color });
return;
}
for (let x = Math.floor(left); x < right; x++) {
const from = xToSeconds(this.view, x) * wave.rate;
const to = xToSeconds(this.view, x + 1) * wave.rate;
const { min, max } = spanPeak(wave.channels, wave.peaks, from, to);
const top = mid - Math.min(1, max) * half;
const bottom = mid - Math.max(-1, min) * half;
this.waveG.rect(x, top, 1, Math.max(1, bottom - top)).fill(color);
}
}
private drawRuler(left: number, right: number): void {
const c = Theme.colors;
const step = rulerStep(this.view.pxPerSecond, LABEL_SPACING);
const placed: { text: string; x: number }[] = [];
const first = Math.ceil(Math.max(0, xToSeconds(this.view, left)) / step);
for (let k = first; ; k++) {
const seconds = k * step;
const x = secondsToX(this.view, seconds);
if (x > right + 0.5) break;
this.overlay.rect(x, RULER_HEIGHT - 8, 1, 7).fill({ color: c.textMuted, alpha: 0.8 });
placed.push({ text: rulerLabel(seconds, step), x: x + 3 });
}
this.syncLabels(placed);
}
/** Dark outside the stretch, a handle at each edge, and the ramp at each edge inside the file. */
private drawCut(asset: SoundAsset, wave: Wave, span: CutSpan, mid: number, half: number): void {
const c = Theme.colors;
const { h } = this;
const a = secondsToX(this.view, span.start);
const b = secondsToX(this.view, span.end);
const shade = { color: 0x000000, alpha: 0.55 };
if (a > 0) this.overlay.rect(0, RULER_HEIGHT, a, h - RULER_HEIGHT).fill(shade);
if (b < this.w) this.overlay.rect(b, RULER_HEIGHT, this.w - b, h - RULER_HEIGHT).fill(shade);
this.overlay.rect(a, 0, Math.max(1, b - a), 4).fill({ color: c.accent, alpha: 0.6 });
const fade = cutOf(resolveAssetProps(this.state.project, asset)).fade;
const range = cutRange(wave.channels[0]!.length, wave.rate, { ...span, fade });
const top = mid - half;
const ramp = (from: number, to: number, rising: boolean) => {
const xa = secondsToX(this.view, from);
const xb = secondsToX(this.view, to);
if (xb - xa < 2) return;
this.overlay
.moveTo(xa, rising ? mid + half : top)
.lineTo(xb, rising ? top : mid + half)
.stroke({ width: 1, color: c.hot, alpha: 0.7 });
};
if (range.fadeIn > 0) ramp(span.start, span.start + range.fadeIn / wave.rate, true);
if (range.fadeOut > 0) ramp(span.end - range.fadeOut / wave.rate, span.end, false);
for (const [x, edge] of [
[a, 'start'],
[b, 'end'],
] as const) {
const held = this.gesture?.kind === 'span' && this.gesture.edge === edge;
this.overlay.rect(x - 1, 0, 2, h).fill(held ? c.hot : c.accent);
const toward = edge === 'start' ? 1 : -1;
this.overlay
.moveTo(x, 0)
.lineTo(x + toward * 8, 0)
.lineTo(x, 10)
.closePath()
.fill(held ? c.hot : c.accent);
}
}
private syncLabels(placed: readonly { text: string; x: number }[]): void {
placed.forEach((label, i) => {
let text = this.texts[i];
if (!text) {
text = crispText({ style: textStyle('small') });
this.texts.push(text);
this.labels.addChild(text);
}
text.visible = true;
if (text.text !== label.text) text.text = label.text;
text.position.set(label.x, 6);
});
for (let i = placed.length; i < this.texts.length; i++) this.texts[i]!.visible = false;
}
}
+188
View File
@@ -0,0 +1,188 @@
import { clamp } from '../../ui';
/**
* The time axis of the Sound tab (#343), apart from Pixi so a test can drive it: seconds run left to
* right, `x` is measured from the left edge of the area that shows the wave.
*/
export interface WaveView {
/** Pixels from the area's left edge to second 0. */
panX: number;
pxPerSecond: number;
}
/** From a few minutes across a screen to a sample every few pixels at 48 kHz. */
export const MIN_PX_PER_SECOND = 0.5;
export const MAX_PX_PER_SECOND = 200_000;
/** How far either end of the file can be panned in from its edge of the area, as a share of its width. */
const LEAD = 0.5;
/** Pixels either side of a handle that grab it. */
export const EDGE_REACH = 6;
export function secondsToX(view: WaveView, seconds: number): number {
return view.panX + seconds * view.pxPerSecond;
}
export function xToSeconds(view: WaveView, x: number): number {
return (x - view.panX) / view.pxPerSecond;
}
/** Zooms about `x`, so the second under the pointer stays under it. */
export function zoomWave(view: WaveView, factor: number, x: number): WaveView {
const pxPerSecond = clamp(view.pxPerSecond * factor, MIN_PX_PER_SECOND, MAX_PX_PER_SECOND);
return { panX: x - xToSeconds(view, x) * pxPerSecond, pxPerSecond };
}
/** The whole file across `width`, a little room either side. */
export function fitWave(duration: number, width: number, margin: number): WaveView {
const room = Math.max(1, width - margin * 2);
const pxPerSecond = clamp(room / Math.max(1e-6, duration), MIN_PX_PER_SECOND, MAX_PX_PER_SECOND);
return { panX: margin, pxPerSecond };
}
/**
* As the music tabs have it across and down: the start of the file no further in than half the
* width, its end no further in from the right – and a file that fits is not pushed off the left.
*/
export function clampWave(view: WaveView, duration: number, width: number): WaveView {
const lead = width * LEAD;
const least = Math.min(0, width - lead - duration * view.pxPerSecond);
return { ...view, panX: clamp(view.panX, least, lead) };
}
/** Seconds between the ruler's labels: 1, 2 or 5 times a power of ten, at least `minPx` apart. */
export function rulerStep(pxPerSecond: number, minPx: number): number {
const rough = minPx / pxPerSecond;
const power = 10 ** Math.floor(Math.log10(rough));
for (const step of [1, 2, 5, 10]) if (step * power >= rough) return step * power;
return 10 * power;
}
/** A second as the ruler writes it, with as many decimals as its step needs. */
export function rulerLabel(seconds: number, step: number): string {
const decimals = Math.max(0, Math.min(6, -Math.floor(Math.log10(step) + 1e-9)));
if (seconds >= 60) {
const minutes = Math.floor(seconds / 60);
const rest = (seconds - minutes * 60).toFixed(decimals);
return `${minutes}:${rest.padStart(decimals > 0 ? decimals + 3 : 2, '0')}`;
}
return `${seconds.toFixed(decimals)}s`;
}
/** The stretch a sound plays, in seconds of its file: `end` is never 0 here. */
export interface CutSpan {
start: number;
end: number;
}
/** The span an asset's start and end mean in a file of `duration`: an end of 0, or past it, is its end. */
export function spanOf(start: number, end: number, duration: number): CutSpan {
const to = end > 0 ? Math.min(end, duration) : duration;
const from = clamp(start, 0, Math.max(0, to));
return { start: from, end: Math.max(from, to) };
}
/** Which handle is at `x`, the nearer one where both are in reach. */
export function edgeAt(view: WaveView, span: CutSpan, x: number): 'start' | 'end' | null {
const toStart = Math.abs(secondsToX(view, span.start) - x);
const toEnd = Math.abs(secondsToX(view, span.end) - x);
if (Math.min(toStart, toEnd) > EDGE_REACH) return null;
return toEnd < toStart || (toEnd === toStart && x > secondsToX(view, span.end)) ? 'end' : 'start';
}
/**
* The span a drag leaves: one edge moved to `at`, or with no edge a new span from where it was
* pressed to `at`. Held inside the file and at least `least` seconds long; an edge dragged past the
* other stops there.
*/
export function draggedSpan(
span: CutSpan,
edge: 'start' | 'end' | null,
from: number,
at: number,
duration: number,
least: number,
): CutSpan {
const inFile = (s: number) => clamp(s, 0, duration);
const gap = Math.min(least, duration);
if (edge === 'start') return { start: Math.min(inFile(at), span.end - gap), end: span.end };
if (edge === 'end') return { start: span.start, end: Math.max(inFile(at), span.start + gap) };
const [a, b] = [inFile(from), inFile(at)].sort((p, q) => p - q) as [number, number];
return b - a >= gap ? { start: a, end: b } : { start: a, end: Math.min(duration, a + gap) };
}
/**
* What an asset stores for a span: seconds, an end at the end of the file as 0 – so a span over the
* whole file is no cut at all, and plays the file itself.
*/
export function storedCut(span: CutSpan, duration: number, rate: number): { start: number; end: number } {
const sample = 1 / rate;
const start = span.start < sample / 2 ? 0 : roundTo(span.start, rate);
const end = duration - span.end < sample / 2 ? 0 : roundTo(span.end, rate);
return { start, end };
}
/** To the nearest sample, written short. */
function roundTo(seconds: number, rate: number): number {
return Number((Math.round(seconds * rate) / rate).toFixed(6));
}
/** Samples each block of `WavePeaks` covers. */
export const PEAK_BLOCK = 256;
/** The lowest and highest sample of each block of a sound, over all its channels. */
export interface WavePeaks {
min: Float32Array;
max: Float32Array;
}
export function wavePeaks(channels: readonly Float32Array[]): WavePeaks {
const length = channels[0]?.length ?? 0;
const blocks = Math.ceil(length / PEAK_BLOCK);
const min = new Float32Array(blocks).fill(Infinity);
const max = new Float32Array(blocks).fill(-Infinity);
for (const data of channels)
for (let i = 0; i < data.length; i++) {
const b = (i / PEAK_BLOCK) | 0;
const v = data[i]!;
if (v < min[b]!) min[b] = v;
if (v > max[b]!) max[b] = v;
}
return { min, max };
}
/**
* The lowest and highest sample from `from` up to `to` (sample indices, `to` after `from`): read
* from the blocks where the stretch spans whole ones, from the samples at its ends.
*/
export function spanPeak(
channels: readonly Float32Array[],
peaks: WavePeaks,
from: number,
to: number,
): { min: number; max: number } {
const length = channels[0]?.length ?? 0;
const a = Math.max(0, Math.floor(from));
const b = Math.min(length, Math.max(a + 1, Math.ceil(to)));
let min = Infinity;
let max = -Infinity;
const raw = (s: number, e: number) => {
for (const data of channels)
for (let i = s; i < e; i++) {
const v = data[i]!;
if (v < min) min = v;
if (v > max) max = v;
}
};
const firstBlock = Math.ceil(a / PEAK_BLOCK);
const lastBlock = Math.floor(b / PEAK_BLOCK);
if (lastBlock <= firstBlock) raw(a, b);
else {
raw(a, firstBlock * PEAK_BLOCK);
for (let k = firstBlock; k < lastBlock; k++) {
if (peaks.min[k]! < min) min = peaks.min[k]!;
if (peaks.max[k]! > max) max = peaks.max[k]!;
}
raw(lastBlock * PEAK_BLOCK, b);
}
return min <= max ? { min, max } : { min: 0, max: 0 };
}
+2 -1
View File
@@ -1,5 +1,6 @@
import type { Asset, SoundFxAsset } from '../../core/model/Asset'; import type { Asset, SoundFxAsset } from '../../core/model/Asset';
import { findAssetOf } from '../../core/model/Project'; import { findAssetOf } from '../../core/model/Project';
import { playedSoundOf } from '../../core/properties/playedSound';
import { resolveAssetProps } from '../../core/properties/resolve'; import { resolveAssetProps } from '../../core/properties/resolve';
import { volumeOf } from '../../core/properties/sounds'; import { volumeOf } from '../../core/properties/sounds';
import { soundFxBuildFor, soundFxParams } from '../../core/soundfx/compile'; import { soundFxBuildFor, soundFxParams } from '../../core/soundfx/compile';
@@ -101,7 +102,7 @@ export class SoundFxEditor extends AssetTabEditor<SoundFxAsset, GraphCanvas> {
const plan = asset && soundFxBuildFor(asset).compiled; const plan = asset && soundFxBuildFor(asset).compiled;
if (!sound?.file || !plan) return; if (!sound?.file || !plan) return;
const volume = volumeOf(resolveAssetProps(this.state.project, sound)); const volume = volumeOf(resolveAssetProps(this.state.project, sound));
const voice = soundLibrary.play(sound.file, volume, false, PREVIEW_OWNER, { const voice = soundLibrary.play(playedSoundOf(this.state.project, sound), volume, false, PREVIEW_OWNER, {
plan, plan,
params: { ...this.values }, params: { ...this.values },
}); });
+7
View File
@@ -803,6 +803,13 @@ export class EditorState {
return findAssetOf(this.project, s.assetId, 'curve')?.id ?? null; return findAssetOf(this.project, s.assetId, 'curve')?.id ?? null;
} }
/** The sound the selection refers to (#343); the stretch it is cut to is set in its tab. */
get selectedSoundId(): string | null {
const s = this.selection;
if (s?.kind !== 'asset') return null;
return findAssetOf(this.project, s.assetId, 'sound')?.id ?? null;
}
/** The pattern the selection refers to (#333); only the asset itself, its notes are picked in the tab. */ /** The pattern the selection refers to (#333); only the asset itself, its notes are picked in the tab. */
get selectedPatternId(): string | null { get selectedPatternId(): string | null {
const s = this.selection; const s = this.selection;
+5 -5
View File
@@ -11,7 +11,7 @@ import { JOINT_CONNECTED_KEY } from '../../core/model/joint';
import { objectReference } from '../../core/model/objectRefs'; import { objectReference } from '../../core/model/objectRefs';
import { findAsset, findAssetOf, findScene, type Project } from '../../core/model/Project'; import { findAsset, findAssetOf, findScene, type Project } from '../../core/model/Project';
import { renderLayerAlong } from '../../core/model/renderLayers'; import { renderLayerAlong } from '../../core/model/renderLayers';
import { soundFileOf } from '../../core/model/sound'; import { playedSoundById, playedSoundOf } from '../../core/properties/playedSound';
import { hasRenderLayer } from '../../core/properties/PropertySchema'; import { hasRenderLayer } from '../../core/properties/PropertySchema';
import { resolveAssetProps } from '../../core/properties/resolve'; import { resolveAssetProps } from '../../core/properties/resolve';
import { loopsOf, volumeOf } from '../../core/properties/sounds'; import { loopsOf, volumeOf } from '../../core/properties/sounds';
@@ -193,14 +193,14 @@ export function behaviorContext(
stopSong: (songId, fadeOut) => run.audio.stopSequence(songVoiceName(songId), false, heldFade(fadeOut)), stopSong: (songId, fadeOut) => run.audio.stopSequence(songVoiceName(songId), false, heldFade(fadeOut)),
stopPlayback: (playback, fadeOut) => run.audio.stopPlayback(playback, heldFade(fadeOut)), stopPlayback: (playback, fadeOut) => run.audio.stopPlayback(playback, heldFade(fadeOut)),
soundLength: (assetId) => { soundLength: (assetId) => {
const file = soundFileOf(project, assetId); const file = playedSoundById(project, assetId);
return file ? run.audio.length(file) : 0; return file ? run.audio.length(file) : 0;
}, },
// An id nothing resolves stops nothing, rather than silencing the whole run by accident; // An id nothing resolves stops nothing, rather than silencing the whole run by accident;
// only the empty id means "everything", which is what the node promises. // only the empty id means "everything", which is what the node promises.
soundTail: (voice) => run.audio.tail(voice), soundTail: (voice) => run.audio.tail(voice),
stopSound: (assetId, letRing) => { stopSound: (assetId, letRing) => {
const file = assetId === '' ? '' : soundFileOf(project, assetId); const file = playedSoundById(project, assetId);
if (assetId === '' || file) run.audio.stop(file, letRing); if (assetId === '' || file) run.audio.stop(file, letRing);
}, },
stopVoice: (voice, letRing) => run.audio.stopVoice(voice, letRing), stopVoice: (voice, letRing) => run.audio.stopVoice(voice, letRing),
@@ -270,7 +270,7 @@ function playSound(
const fx = fxId ? findAssetOf(project, fxId, 'soundfx') : undefined; const fx = fxId ? findAssetOf(project, fxId, 'soundfx') : undefined;
const plan = fx && soundFxBuildFor(fx).compiled; const plan = fx && soundFxBuildFor(fx).compiled;
return audio.play( return audio.play(
asset.file, playedSoundOf(project, asset),
level, level,
loop ?? loopsOf(props), loop ?? loopsOf(props),
owner, owner,
@@ -404,7 +404,7 @@ export function playClip(
Math.min(dbToGain(SOUND_ITEM_MAX_DB), volumeOf(resolveAssetProps(project, asset)) * clip.volume), Math.min(dbToGain(SOUND_ITEM_MAX_DB), volumeOf(resolveAssetProps(project, asset)) * clip.volume),
); );
return audio.playClip( return audio.playClip(
asset.file, playedSoundOf(project, asset),
{ at: clip.at, offset: clip.offset, duration: clip.duration, volume, bus: clip.bus }, { at: clip.at, offset: clip.offset, duration: clip.duration, volume, bus: clip.bus },
clipFx(project, clip.fx, clip.tempo), clipFx(project, clip.fx, clip.tempo),
); );
+101 -15
View File
@@ -21,6 +21,7 @@ import { type BusLevel, Playbacks, type SequencePlay } from '../core/music/playb
import type { ScheduledClip, ScheduledNote } from '../core/music/Scheduler'; import type { ScheduledClip, ScheduledNote } from '../core/music/Scheduler';
import { dbToGain, SOUND_ITEM_MAX_DB } from '../core/music/song'; import { dbToGain, SOUND_ITEM_MAX_DB } from '../core/music/song';
import { DEFAULT_TEMPO } from '../core/music/time'; import { DEFAULT_TEMPO } from '../core/music/time';
import { type CutRange, cutChannels, cutRange, parseCutName } from '../core/model/sound';
import type { AudioSink, ClipTiming, NoteTiming } from '../runtime/systems/Audio'; import type { AudioSink, ClipTiming, NoteTiming } from '../runtime/systems/Audio';
import type { PitchOptions } from './pitchWorklet'; import type { PitchOptions } from './pitchWorklet';
import pitchWorklet from './pitchWorklet.ts?worker&url'; import pitchWorklet from './pitchWorklet.ts?worker&url';
@@ -39,6 +40,8 @@ const DECODE_RATE = 48000;
const FADE = 0.01; const FADE = 0.01;
/** How finely `peaks` cuts a sound up. */ /** How finely `peaks` cuts a sound up. */
const PEAK_BUCKETS = 1024; const PEAK_BUCKETS = 1024;
/** Samples the cuts made so far may hold together before the oldest are let go; one is always kept. */
const CUT_BUDGET = 64 * 1024 * 1024;
/** /**
* One number of a voice that can still change while it plays (#315): a level, a share of a Mix, the * One number of a voice that can still change while it plays (#315): a level, a share of a Mix, the
@@ -221,6 +224,12 @@ export class SoundLibrary implements AudioSink {
private context: AudioContext | null = null; private context: AudioContext | null = null;
private offline: OfflineAudioContext | null = null; private offline: OfflineAudioContext | null = null;
private readonly buffers = new Map<string, AudioBuffer>(); private readonly buffers = new Map<string, AudioBuffer>();
/**
* The stretches of files that sound assets are cut to (#343), made from the whole file on first
* play, oldest first. Unlike a file they can be made again at any time, so they are let go past a
* budget: a cut dragged about in its tab would otherwise leave a copy of the sound per try.
*/
private readonly cuts = new Map<string, AudioBuffer>();
/** The pitch worklet being loaded, and whether it is there once it is known; null before that. */ /** The pitch worklet being loaded, and whether it is there once it is known; null before that. */
private pitching: Promise<boolean> | null = null; private pitching: Promise<boolean> | null = null;
private canPitch: boolean | null = null; private canPitch: boolean | null = null;
@@ -286,7 +295,7 @@ export class SoundLibrary implements AudioSink {
* same frame it asked for it, and the voice then starts with those values. * same frame it asked for it, and the voice then starts with those values.
*/ */
play(file: string, volume: number, loop: boolean, owner = '', fx: FxVoice = DRY_VOICE): number { play(file: string, volume: number, loop: boolean, owner = '', fx: FxVoice = DRY_VOICE): number {
if (file === '' || this.failed.has(file)) return 0; if (file === '' || this.hasFailed(file)) return 0;
const layered = { layers: [fx.plan], params: fx.params, tempo: fx.tempo ?? DEFAULT_TEMPO }; const layered = { layers: [fx.plan], params: fx.params, tempo: fx.tempo ?? DEFAULT_TEMPO };
return this.begin({ id: 0, file, files: [file], volume, loop, owner, fx: layered, duration: 0, at: 0 }); return this.begin({ id: 0, file, files: [file], volume, loop, owner, fx: layered, duration: 0, at: 0 });
} }
@@ -307,7 +316,7 @@ export class SoundLibrary implements AudioSink {
timing: NoteTiming = {}, timing: NoteTiming = {},
): number { ): number {
const files = [...new Set(layers.flatMap(filesOf))].filter( const files = [...new Set(layers.flatMap(filesOf))].filter(
(file) => file !== '' && !this.failed.has(file), (file) => file !== '' && !this.hasFailed(file),
); );
if (files.length === 0) return 0; if (files.length === 0) return 0;
const fx = { layers, params, tempo: timing.tempo ?? DEFAULT_TEMPO }; const fx = { layers, params, tempo: timing.tempo ?? DEFAULT_TEMPO };
@@ -322,7 +331,7 @@ export class SoundLibrary implements AudioSink {
* gets its bytes late starts as far into the sound as it would be by then. * gets its bytes late starts as far into the sound as it would be by then.
*/ */
playClip(file: string, clip: ClipTiming, fx: FxVoice = DRY_VOICE): number { playClip(file: string, clip: ClipTiming, fx: FxVoice = DRY_VOICE): number {
if (file === '' || this.failed.has(file)) return 0; if (file === '' || this.hasFailed(file)) return 0;
const layered = { layers: [fx.plan], params: fx.params, tempo: fx.tempo ?? DEFAULT_TEMPO }; const layered = { layers: [fx.plan], params: fx.params, tempo: fx.tempo ?? DEFAULT_TEMPO };
const { at, offset, duration, volume, bus } = clip; const { at, offset, duration, volume, bus } = clip;
return this.begin({ return this.begin({
@@ -537,7 +546,7 @@ export class SoundLibrary implements AudioSink {
private begin(start: Start): number { private begin(start: Start): number {
const id = this.nextId++; const id = this.nextId++;
const asked: Start = { ...start, id }; const asked: Start = { ...start, id };
const loaded = start.files.every((file) => this.buffers.has(file)); const loaded = start.files.every((file) => this.buffer(file) !== null);
const pitching = start.fx.layers.some(pitches) ? this.pitchWorklet() : null; const pitching = start.fx.layers.some(pitches) ? this.pitchWorklet() : null;
if (loaded && (!pitching || this.canPitch !== null)) { if (loaded && (!pitching || this.canPitch !== null)) {
this.start(asked); this.start(asked);
@@ -598,6 +607,11 @@ export class SoundLibrary implements AudioSink {
return voice ? this.tailOf(voice.fx) : 0; return voice ? this.tailOf(voice.fx) : 0;
} }
/** The second of the audio clock a voice started at; null while it waits on its bytes or is gone. */
startedAt(id: number): number | null {
return this.voices.find((voice) => voice.id === id)?.at ?? null;
}
/** Whether a voice is still heard – or waiting on its bytes to be. A pattern or song counts as one. */ /** Whether a voice is still heard – or waiting on its bytes to be. A pattern or song counts as one. */
playing(id: number): boolean { playing(id: number): boolean {
return this.sounding(id) || this.sequences.has(id); return this.sounding(id) || this.sequences.has(id);
@@ -743,9 +757,23 @@ export class SoundLibrary implements AudioSink {
return pitchWorklet; return pitchWorklet;
} }
/** How long a decoded sound runs, in seconds; null while it has not been loaded (test hook). */ /**
* How long a decoded sound runs, in seconds – a cut (#343) as long as what it keeps; null while it
* has not been loaded (test hook).
*/
duration(file: string): number | null { duration(file: string): number | null {
return this.buffers.get(file)?.duration ?? null; const range = this.rangeOf(file);
return range ? (range.to - range.from) / range.whole.sampleRate : null;
}
/** The decoded samples of a file as it is, without a cut; null while it has not been loaded. */
decoded(file: string): AudioBuffer | null {
return this.buffers.get(file) ?? null;
}
/** Whether a file could not be fetched or decoded – it is not tried again. */
failedToLoad(file: string): boolean {
return this.hasFailed(file);
} }
length(file: string): number { length(file: string): number {
@@ -759,13 +787,14 @@ export class SoundLibrary implements AudioSink {
peaks(file: string): Float32Array | null { peaks(file: string): Float32Array | null {
const known = this.peaked.get(file); const known = this.peaked.get(file);
if (known) return known; if (known) return known;
const buffer = this.buffers.get(file); const range = this.rangeOf(file);
if (!buffer) return null; if (!range) return null;
const { whole, from, to } = range;
const peaks = new Float32Array(PEAK_BUCKETS); const peaks = new Float32Array(PEAK_BUCKETS);
for (let c = 0; c < buffer.numberOfChannels; c++) { for (let c = 0; c < whole.numberOfChannels; c++) {
const data = buffer.getChannelData(c); const data = whole.getChannelData(c);
for (let i = 0; i < data.length; i++) { for (let i = from; i < to; i++) {
const bucket = Math.floor((i / data.length) * PEAK_BUCKETS); const bucket = Math.floor(((i - from) / (to - from)) * PEAK_BUCKETS);
peaks[bucket] = Math.max(peaks[bucket]!, Math.abs(data[i]!)); peaks[bucket] = Math.max(peaks[bucket]!, Math.abs(data[i]!));
} }
} }
@@ -782,8 +811,15 @@ export class SoundLibrary implements AudioSink {
for (const file of files) if (file) void this.load(file); for (const file of files) if (file) void this.load(file);
} }
/** Fetches and decodes once per file; a second play while it is in flight waits on the first. */ /**
* Fetches and decodes once per file; a second play while it is in flight waits on the first. A cut
* (#343) loads its file and is made from it.
*/
async load(file: string): Promise<AudioBuffer | null> { async load(file: string): Promise<AudioBuffer | null> {
if (parseCutName(file).cut) {
const whole = await this.load(parseCutName(file).file);
return whole ? this.buffer(file) : null;
}
const held = this.buffers.get(file); const held = this.buffers.get(file);
if (held) return held; if (held) return held;
if (this.failed.has(file)) return null; if (this.failed.has(file)) return null;
@@ -934,7 +970,13 @@ export class SoundLibrary implements AudioSink {
const wiring: Wiring = { const wiring: Wiring = {
context, context,
at: when, at: when,
buffers: this.buffers, // Held here, not looked up later: a cut can be let go while its voice plays.
buffers: new Map(
files.flatMap((name) => {
const buffer = this.buffer(name);
return buffer ? [[name, buffer] as const] : [];
}),
),
file, file,
loop, loop,
rings, rings,
@@ -957,7 +999,7 @@ export class SoundLibrary implements AudioSink {
layers: fx.layers.length, layers: fx.layers.length,
pitches: wiring.pitches, pitches: wiring.pitches,
stages: wiring.stages, stages: wiring.stages,
duration: Math.max(0, ...files.map((name) => this.buffers.get(name)?.duration ?? 0)), duration: Math.max(0, ...files.map((name) => this.duration(name) ?? 0)),
at: when, at: when,
envelopes: wiring.envelopes.size, envelopes: wiring.envelopes.size,
}; };
@@ -1173,6 +1215,50 @@ export class SoundLibrary implements AudioSink {
return (this.context ??= new AudioContext()); return (this.context ??= new AudioContext());
} }
/** What a name plays, a cut made from its file the first time; null while the file is not loaded. */
private buffer(name: string): AudioBuffer | null {
const whole = this.buffers.get(name);
if (whole) return whole;
const held = this.cuts.get(name);
if (held) {
this.cuts.delete(name);
this.cuts.set(name, held);
return held;
}
const range = this.rangeOf(name);
if (!range) return null;
const made = this.decoder().createBuffer(
range.whole.numberOfChannels,
range.to - range.from,
range.whole.sampleRate,
);
const channels = Array.from({ length: made.numberOfChannels }, (_, c) => range.whole.getChannelData(c));
cutChannels(channels, range).forEach((data, c) => made.copyToChannel(data, c));
this.cuts.set(name, made);
const samples = (buffer: AudioBuffer) => buffer.length * buffer.numberOfChannels;
let total = [...this.cuts.values()].reduce((sum, buffer) => sum + samples(buffer), 0);
for (const [other, buffer] of this.cuts) {
if (total <= CUT_BUDGET || other === name) break;
this.cuts.delete(other);
total -= samples(buffer);
}
return made;
}
/** The samples of its loaded file a name plays: all of them, or the stretch a cut keeps. */
private rangeOf(name: string): (CutRange & { whole: AudioBuffer }) | null {
const { file, cut } = parseCutName(name);
const whole = this.buffers.get(file);
if (!whole) return null;
if (!cut) return { whole, from: 0, to: whole.length, fadeIn: 0, fadeOut: 0 };
return { whole, ...cutRange(whole.length, whole.sampleRate, cut) };
}
/** Whether the file of a name could not be had: a cut of it cannot either. */
private hasFailed(name: string): boolean {
return this.failed.has(parseCutName(name).file);
}
/** Loads the pitch worklet into the context once; says whether it is there. */ /** Loads the pitch worklet into the context once; says whether it is there. */
private pitchWorklet(): Promise<boolean> { private pitchWorklet(): Promise<boolean> {
return (this.pitching ??= (async () => { return (this.pitching ??= (async () => {
+83 -3
View File
@@ -11,10 +11,27 @@ import {
} from '../../src/core/commands/assetCommands'; } from '../../src/core/commands/assetCommands';
import type { BehaviorAsset, SoundAsset } from '../../src/core/model/Asset'; import type { BehaviorAsset, SoundAsset } from '../../src/core/model/Asset';
import { isPlaceableType } from '../../src/core/model/AssetTypes'; import { isPlaceableType } from '../../src/core/model/AssetTypes';
import { isSoundFile, soundFileOf } from '../../src/core/model/sound'; import {
cutChannels,
cutName,
cutRange,
isSoundFile,
parseCutName,
soundFileOf,
} from '../../src/core/model/sound';
import { assetRefChoices } from '../../src/core/properties/assetRef'; import { assetRefChoices } from '../../src/core/properties/assetRef';
import { schemaFor } from '../../src/core/properties/PropertySchema'; import { schemaFor } from '../../src/core/properties/PropertySchema';
import { LOOP_KEY, loopsOf, VOLUME_KEY, volumeOf } from '../../src/core/properties/sounds'; import { playedSoundById, playedSoundOf } from '../../src/core/properties/playedSound';
import {
cutOf,
END_KEY,
FADE_KEY,
LOOP_KEY,
loopsOf,
START_KEY,
VOLUME_KEY,
volumeOf,
} from '../../src/core/properties/sounds';
import { deserializeProject } from '../../src/core/store/serialize'; import { deserializeProject } from '../../src/core/store/serialize';
import type { AudioSink } from '../../src/runtime/systems/Audio'; import type { AudioSink } from '../../src/runtime/systems/Audio';
import { import {
@@ -99,7 +116,14 @@ describe('the sound asset', () => {
expect(asset).toMatchObject({ type: 'sound', file: '' }); expect(asset).toMatchObject({ type: 'sound', file: '' });
expect(volumeOf(asset.props)).toBe(1); expect(volumeOf(asset.props)).toBe(1);
expect(loopsOf(asset.props)).toBe(false); expect(loopsOf(asset.props)).toBe(false);
expect(schemaFor('sound').map((def) => def.key)).toEqual([VOLUME_KEY, LOOP_KEY]); expect(schemaFor('sound').map((def) => def.key)).toEqual([
VOLUME_KEY,
LOOP_KEY,
START_KEY,
END_KEY,
FADE_KEY,
]);
expect(cutOf(asset.props)).toEqual({ start: 0, end: 0, fade: 0.01 });
}); });
it('refuses a file name the server could not have produced', () => { it('refuses a file name the server could not have produced', () => {
@@ -142,6 +166,62 @@ describe('the sound asset', () => {
}); });
}); });
describe('the cut of a sound (#343)', () => {
it('is no cut at all over the whole file, and a name of its own otherwise', () => {
const whole = { start: 0, end: 0, fade: 0.01 };
expect(cutName(FILE, whole)).toBe(FILE);
expect(cutName('', { ...whole, start: 1 })).toBe('');
const name = cutName(FILE, { start: 0.25, end: 1.5, fade: 0.01 });
expect(name).toBe(`${FILE}#0.25~1.5~0.01`);
expect(parseCutName(name)).toEqual({ file: FILE, cut: { start: 0.25, end: 1.5, fade: 0.01 } });
expect(parseCutName(FILE)).toEqual({ file: FILE, cut: null });
expect(parseCutName(`${FILE}#x~-2`)).toEqual({ file: FILE, cut: { start: 0, end: 0, fade: 0 } });
});
it('keeps at least a sample inside the file, and ramps only the edges inside it', () => {
// 10 samples at 10 Hz: one sample per tenth of a second.
expect(cutRange(10, 10, { start: 0.2, end: 0.8, fade: 0.2 })).toEqual({
from: 2,
to: 8,
fadeIn: 2,
fadeOut: 2,
});
expect(cutRange(10, 10, { start: 0, end: 0.5, fade: 0.2 })).toMatchObject({ fadeIn: 0, fadeOut: 2 });
expect(cutRange(10, 10, { start: 0.5, end: 0, fade: 0.2 })).toMatchObject({
to: 10,
fadeIn: 2,
fadeOut: 0,
});
// Past the end, an end before the start, and a fade longer than half of what is kept.
expect(cutRange(10, 10, { start: 5, end: 0, fade: 0 })).toMatchObject({ from: 9, to: 10 });
expect(cutRange(10, 10, { start: 0.6, end: 0.3, fade: 0 })).toMatchObject({ from: 6, to: 7 });
expect(cutRange(10, 10, { start: 0.2, end: 0.6, fade: 1 })).toMatchObject({ fadeIn: 2, fadeOut: 2 });
});
it('cuts every channel alike, rising from silence and falling to it at an inner edge', () => {
const left = new Float32Array(10).fill(1);
const right = new Float32Array(10).fill(-0.5);
const [l, r] = cutChannels([left, right], { from: 2, to: 8, fadeIn: 2, fadeOut: 2 });
expect(Array.from(l!)).toEqual([0, 0.5, 1, 1, 0.5, 0]);
expect(Array.from(r!)).toEqual([-0, -0.25, -0.5, -0.5, -0.25, -0]);
expect(Array.from(left)).toEqual(new Array(10).fill(1));
});
it('is what every way of playing the asset is given, its fade in milliseconds on the asset', () => {
const f = playingScene();
expect(playedSoundOf(f.project, f.sound)).toBe(FILE);
f.stack.execute(new SetAssetPropertyCommand(f.sound.id, START_KEY, 0.5));
f.stack.execute(new SetAssetPropertyCommand(f.sound.id, FADE_KEY, 25));
const name = `${FILE}#0.5~0~0.025`;
expect(playedSoundOf(f.project, f.sound)).toBe(name);
expect(playedSoundById(f.project, f.sound.id)).toBe(name);
expect(playedSoundById(f.project, '')).toBe('');
const audio = fakeAudio();
startWorld(f, { behaviors: true, audio }).frame();
expect(audio.played).toMatchObject([{ file: name }]);
});
});
describe('Play sound', () => { describe('Play sound', () => {
it('plays the asset as the project describes it', () => { it('plays the asset as the project describes it', () => {
const f = playingScene(); const f = playingScene();
+101
View File
@@ -0,0 +1,101 @@
/**
* The Sound tab's view (#343): seconds across, zoomed about the pointer, and the stretch of the
* file between two handles – how a drag moves it and what the asset then stores.
*/
import { describe, expect, it } from 'vitest';
import {
clampWave,
draggedSpan,
edgeAt,
fitWave,
rulerLabel,
rulerStep,
secondsToX,
spanOf,
spanPeak,
storedCut,
wavePeaks,
xToSeconds,
zoomWave,
} from '../../src/editor/sound/waveView';
describe('the wave view', () => {
it('zooms about the pointer and fits the file with a margin either side', () => {
const view = { panX: 10, pxPerSecond: 100 };
const zoomed = zoomWave(view, 2, 210);
expect(xToSeconds(zoomed, 210)).toBeCloseTo(xToSeconds(view, 210));
expect(zoomed.pxPerSecond).toBe(200);
const fitted = fitWave(4, 832, 16);
expect(secondsToX(fitted, 0)).toBe(16);
expect(secondsToX(fitted, 4)).toBe(816);
});
it('keeps either end of the file from going further in than half the width', () => {
expect(clampWave({ panX: 900, pxPerSecond: 100 }, 20, 1000).panX).toBe(500);
expect(clampWave({ panX: -1600, pxPerSecond: 100 }, 20, 1000).panX).toBe(-1500);
expect(clampWave({ panX: -900, pxPerSecond: 100 }, 20, 1000).panX).toBe(-900);
// Two seconds fit: they stay on screen.
expect(clampWave({ panX: 20, pxPerSecond: 100 }, 2, 1000).panX).toBe(20);
expect(clampWave({ panX: -50, pxPerSecond: 100 }, 2, 1000).panX).toBe(0);
});
it('labels the ruler in steps of 1, 2 or 5, with the decimals the step needs', () => {
expect(rulerStep(100, 70)).toBe(1);
expect(rulerStep(40, 70)).toBe(2);
expect(rulerStep(1000, 70)).toBeCloseTo(0.1);
expect(rulerStep(10, 70)).toBe(10);
expect(rulerLabel(1.5, 0.5)).toBe('1.5s');
expect(rulerLabel(2, 1)).toBe('2s');
expect(rulerLabel(0.25, 0.05)).toBe('0.25s');
expect(rulerLabel(75, 5)).toBe('1:15');
});
});
describe('the cut span', () => {
const view = { panX: 0, pxPerSecond: 100 };
it('reads an end of 0 or past the file as its end', () => {
expect(spanOf(0, 0, 3)).toEqual({ start: 0, end: 3 });
expect(spanOf(1, 9, 3)).toEqual({ start: 1, end: 3 });
expect(spanOf(2.5, 1, 3)).toEqual({ start: 1, end: 1 });
});
it('finds the handle in reach, the nearer one where both are', () => {
const span = { start: 1, end: 2 };
expect(edgeAt(view, span, 103)).toBe('start');
expect(edgeAt(view, span, 197)).toBe('end');
expect(edgeAt(view, span, 150)).toBeNull();
expect(edgeAt(view, { start: 1, end: 1.04 }, 103)).toBe('end');
});
it('moves one edge, never past the other, or spans a new stretch inside the file', () => {
const span = { start: 1, end: 2 };
expect(draggedSpan(span, 'start', 1, 0.5, 3, 0.001)).toEqual({ start: 0.5, end: 2 });
expect(draggedSpan(span, 'start', 1, 2.5, 3, 0.001)).toEqual({ start: 1.999, end: 2 });
expect(draggedSpan(span, 'end', 2, 9, 3, 0.001)).toEqual({ start: 1, end: 3 });
expect(draggedSpan(span, null, 2.5, 0.5, 3, 0.001)).toEqual({ start: 0.5, end: 2.5 });
expect(draggedSpan(span, null, -1, 1, 3, 0.001)).toEqual({ start: 0, end: 1 });
});
it('stores the ends of the file as 0, everything else to the sample', () => {
expect(storedCut({ start: 0, end: 3 }, 3, 48000)).toEqual({ start: 0, end: 0 });
expect(storedCut({ start: 0.000001, end: 2.999999 }, 3, 48000)).toEqual({ start: 0, end: 0 });
expect(storedCut({ start: 1.2345678, end: 2 }, 3, 48000)).toEqual({ start: 1.234563, end: 2 });
});
});
describe('the wave peaks', () => {
it('reads the lowest and highest sample of any stretch, over the blocks and the samples at its ends', () => {
const left = new Float32Array(1000);
const right = new Float32Array(1000);
left[10] = 0.5;
right[600] = -0.75;
left[999] = 0.9;
const peaks = wavePeaks([left, right]);
expect(peaks.max).toHaveLength(4);
expect(spanPeak([left, right], peaks, 0, 1000)).toEqual({ min: -0.75, max: Math.fround(0.9) });
expect(spanPeak([left, right], peaks, 5, 20)).toEqual({ min: 0, max: 0.5 });
expect(spanPeak([left, right], peaks, 11, 599)).toEqual({ min: 0, max: 0 });
expect(spanPeak([left, right], peaks, 300.2, 300.4)).toEqual({ min: 0, max: 0 });
});
});
+18
View File
@@ -159,6 +159,7 @@ function fakeBuffer(_channels: number, length: number, sampleRate: number) {
sampleRate, sampleRate,
duration: length / sampleRate, duration: length / sampleRate,
getChannelData: () => data, getChannelData: () => data,
copyToChannel: (from: Float32Array) => data.set(from),
}; };
} }
@@ -378,6 +379,23 @@ describe('the sound library', () => {
expect(sources.filter((s) => s.started)).toHaveLength(2); expect(sources.filter((s) => s.started)).toHaveLength(2);
}); });
it('plays the stretch a sound is cut to, made from its file without fetching it again (#343)', async () => {
const library = new SoundLibrary();
// The file is 1200 samples at 800 Hz: 1.5 s. The cut keeps 0.5 s to 1 s.
const cut = `${FILE}#0.5~1~0.01`;
await play(library, cut);
expect(library.stats()).toMatchObject({ started: 1, decoded: 1, playing: 1 });
expect((sources[0]?.buffer as { length: number }).length).toBe(400);
expect(library.duration(cut)).toBe(0.5);
expect(library.duration(FILE)).toBe(1.5);
expect(library.peaks(cut)).toHaveLength(1024);
// The file itself plays whole beside it, and a stop by the cut's name reaches only the cut.
await play(library);
expect((sources[1]?.buffer as { length: number }).length).toBe(1200);
library.stop(cut);
expect(library.stats().playing).toBe(1);
});
it('loads ahead of the first play without opening the device, so that play starts at once', async () => { it('loads ahead of the first play without opening the device, so that play starts at once', async () => {
const opened: string[] = []; const opened: string[] = [];
vi.stubGlobal( vi.stubGlobal(
+102
View File
@@ -1152,6 +1152,108 @@ export async function sound(t) {
JSON.stringify({ dropped, both, notice: await json('__gm.state.notice') }), JSON.stringify({ dropped, both, notice: await json('__gm.state.notice') }),
); );
// 34h-2. The Sound tab (#343): the drone's 0.75 s as a wave between two handles. Dragging the end
// handle cuts the sound there, a drag beside the handles spans a new stretch, and that
// stretch is what plays: a name of its own, its samples cut to the sample and ramped from
// silence at an edge inside the file. One drag is one undo step.
const droneId = both.find((a) => a.file === drone.file)?.id;
await evaluate(`__gm.state.openAsset(${JSON.stringify(droneId)})`);
const tab = () => json('__gm.soundTab.span()');
await waitFor(tab, (span) => span !== null);
const tabOpened = {
current: await json('__gm.soundTab.current()'),
span: await tab(),
info: await json('__gm.soundTab.info()'),
};
check(
'opening a sound shows its tab: the whole file between the handles',
tabOpened.current === droneId &&
tabOpened.span?.start === 0 &&
Math.abs(tabOpened.span?.end - 0.75) < 0.001 &&
tabOpened.info === '0.75 s',
JSON.stringify(tabOpened),
);
const droneProps = async () => (await assetsOfType('sound')).find((a) => a.id === droneId)?.props;
let dragFrom = await json('__gm.soundTab.at(0.75)');
let dragTo = await json('__gm.soundTab.at(0.5)');
await t.mouse('mouseMoved', dragFrom.x - 1, dragFrom.y);
await t.mouse('mouseMoved', dragFrom.x, dragFrom.y);
const onHandle = await json('__gm.soundTab.cursor()');
await t.drag(dragFrom.x, dragFrom.y, dragTo.x, dragTo.y);
const shortened = { props: await droneProps(), info: await json('__gm.soundTab.info()') };
check(
'dragging the end handle cuts the sound there, the handle showing it can be dragged',
onHandle === 'ew-resize' &&
shortened.props?.start === 0 &&
Math.abs(shortened.props?.end - 0.5) < 0.01 &&
shortened.info === `${shortened.props?.end.toFixed(2)} s of 0.75 s`,
JSON.stringify({ onHandle, ...shortened }),
);
await press('z', { ctrl: true });
const undoneCut = await droneProps();
await press('z', { ctrl: true, shift: true });
check(
'one undo puts the whole file back, a redo the cut',
(undoneCut?.end ?? 0) === 0 && Math.abs((await droneProps())?.end - 0.5) < 0.01,
JSON.stringify({ undoneCut, redone: await droneProps() }),
);
dragFrom = await json('__gm.soundTab.at(0.1)');
dragTo = await json('__gm.soundTab.at(0.3)');
await t.drag(dragFrom.x, dragFrom.y, dragTo.x, dragTo.y);
const spanned = await droneProps();
const cutPlayed = await json(`__gm.soundTab.played(${JSON.stringify(droneId)})`);
const cutSamples = await evaluate(`__gm.soundTab.samples(${JSON.stringify(cutPlayed)})`);
const keptSamples = spanned ? Math.round((spanned.end - spanned.start) * (cutSamples?.rate ?? 0)) : -1;
check(
'a drag beside the handles spans a new stretch, played by a name of its own, cut to the sample',
Math.abs(spanned?.start - 0.1) < 0.01 &&
Math.abs(spanned?.end - 0.3) < 0.01 &&
cutPlayed === `${drone.file}#${spanned?.start}~${spanned?.end}~0.01` &&
cutSamples?.length === keptSamples,
JSON.stringify({ spanned, cutPlayed, cutSamples, keptSamples }),
);
check(
'the cut rises from silence and falls back to it, the default 10 ms at each edge inside the file',
cutSamples?.head[0] === 0 && Math.abs(cutSamples?.tail.at(-1)) < 0.01,
JSON.stringify(cutSamples),
);
const tabFade = await json('__gm.soundTab.fadeField()');
await click(tabFade.x, tabFade.y);
await t.typeText('0');
await press('Enter');
await sleep(150);
const unfaded = await json(`__gm.soundTab.played(${JSON.stringify(droneId)})`);
check(
'the header sets the fade at the edges, in milliseconds on the asset',
(await droneProps())?.fade === 0 && unfaded.endsWith('~0'),
JSON.stringify({ props: await droneProps(), unfaded }),
);
await evaluate('__gm.sound.stopAll()');
const tabPlay = await json('__gm.soundTab.playButton()');
await click(tabPlay.x, tabPlay.y);
const tabHeard = await waitFor(stats, (s) => s.last?.file === unfaded);
const tabPlayhead = await waitFor(
() => json('__gm.soundTab.playhead()'),
(seconds) => seconds !== null && seconds > 0.1,
10,
);
const tabEnded = await waitFor(
() => json('__gm.soundTab.playing()'),
(on) => on === false,
);
check(
'Play plays only the stretch, its playhead running inside it, and stops when it ends',
tabHeard.last?.file === unfaded &&
Math.abs(tabHeard.last?.duration - 0.2) < 0.01 &&
tabPlayhead !== null &&
tabPlayhead >= 0.1 &&
tabPlayhead <= 0.3 &&
tabEnded === false &&
(await json('__gm.soundTab.playhead()')) === null,
JSON.stringify({ last: tabHeard.last, tabPlayhead, tabEnded }),
);
await shot('34-sound-tab');
// 34i. The step takes back everything it made, so the steps after it see the fixture again. // 34i. The step takes back everything it made, so the steps after it see the fixture again.
await mutate(` await mutate(`
const gone = new Set(['sn-logic', ...${JSON.stringify(both.map((a) => a.id))}]); const gone = new Set(['sn-logic', ...${JSON.stringify(both.map((a) => a.id))}]);