From de14f0f9e272d13588df5ae889ac8d59a65ef3f5 Mon Sep 17 00:00:00 2001 From: MelekTaus Date: Mon, 5 Oct 2026 16:45:42 +0200 Subject: [PATCH] Play a sound through a graph of gain, pitch and EQ effects #311 A sound could only be played as it was uploaded. A Sound FX asset is a node graph like a shader: Input, effects, Output. Play sound has a new input fx that sends the voice through it. - New asset type soundfx ("Sound FX"), not placeable, with a graph and its own tab. The heading in the asset panel is "Sound effects", so it does not take the click meant for the + menu entry. - Nodes in core/soundfx/nodes.ts, category sound, port type audio that only fits audio: Input, Output, Gain (dB), Pitch (semitones), EQ (80 Hz and 12 kHz shelves, peaks at 250 Hz, 1 kHz and 4 kHz). - compileSoundFx walks back from the Output with the shared graph emitter. An input takes one edge, so the result is one line: a pitch and a list of gain and filter stages. A node without input or a graph without Output reports a problem on the node. - Pitch is the playback rate of the source. It applies to the whole sound and also changes its length, the node description says so. - The BehaviorSystem resolves fx and hands the line to the AudioSink. An fx that is missing or does not compile plays the sound dry. - SoundLibrary wires source, stages, volume, destination in that order. stats().last now also names pitch and stage count. - The tab header picks one of the project's sounds and plays it through the graph as it stands. Stop there ends only its own voice. Tests: - soundfx.test.ts (new, 11), sound-library (9, +2 for the wiring), node-catalog (+1 for fx), graph and asset-groups adjusted - e2e 34: ALL PASSED - 26 checks (+5: + menu opens the tab, palette, 2 effects, tab Play and game Play through the FX at -12 st) - Gate on melektaus-atx: typecheck, lint, prettier clean, 239 of 240 test files green. The red one was sound-library (its fake AudioContext had no playbackRate), fixed afterwards, gate not re-run in full. - Counter-check: without the audio rule in portsCompatible the port test fails. Co-Authored-By: Claude Opus 5.5 --- src/core/commands/assetCommands.ts | 2 + src/core/graph/Graph.ts | 16 +- src/core/graph/NodeContext.ts | 3 +- src/core/graph/NodeRegistry.ts | 15 +- src/core/graph/nodes/action/PlaySound.ts | 14 +- src/core/graph/nodes/index.ts | 2 + src/core/graph/nodes/shared.ts | 8 + src/core/model/Asset.ts | 13 +- src/core/model/AssetTypes.ts | 7 +- src/core/properties/PropertySchema.ts | 2 + src/core/soundfx/compile.ts | 67 +++++++ src/core/soundfx/nodes.ts | 147 +++++++++++++++ src/core/store/serialize.ts | 3 + src/editor/Editor.ts | 9 + src/editor/assetTabs.ts | 1 + src/editor/debug/graphHooks.ts | 14 ++ src/editor/graph/GraphCanvas.ts | 5 +- src/editor/graph/NodeView.ts | 4 + src/editor/graph/paletteEntries.ts | 12 +- src/editor/panels/assetGroups.ts | 2 + src/editor/panels/assetIcons.ts | 2 + src/editor/panels/icons/soundfx.svg | 7 + src/editor/soundfx/SoundFxEditor.ts | 107 +++++++++++ src/editor/state.ts | 5 + src/runtime/systems/Audio.ts | 7 +- src/runtime/systems/behaviorContext.ts | 13 +- src/ui/Theme.ts | 2 + src/viewport/SoundLibrary.ts | 52 +++++- tests/core/graph.test.ts | 6 +- tests/core/node-catalog.test.ts | 8 +- tests/core/soundfx.test.ts | 219 +++++++++++++++++++++++ tests/helpers.ts | 5 +- tests/viewport/sound-library.test.ts | 77 +++++++- tools/e2e/34-sound.mjs | 86 +++++++++ 34 files changed, 899 insertions(+), 43 deletions(-) create mode 100644 src/core/soundfx/compile.ts create mode 100644 src/core/soundfx/nodes.ts create mode 100644 src/editor/panels/icons/soundfx.svg create mode 100644 src/editor/soundfx/SoundFxEditor.ts create mode 100644 tests/core/soundfx.test.ts diff --git a/src/core/commands/assetCommands.ts b/src/core/commands/assetCommands.ts index a0fc9a1f..8e9bcb5f 100644 --- a/src/core/commands/assetCommands.ts +++ b/src/core/commands/assetCommands.ts @@ -101,6 +101,8 @@ export function buildAsset(type: AssetType, name: string, id = newId()): Asset { return { ...base, type: 'font', file: '' }; case 'sound': return { ...base, type: 'sound', file: '' }; + case 'soundfx': + return { ...base, type: 'soundfx', graph: createEmptyGraph() }; case 'particles': return { ...base, type: 'particles', graph: createEmptyGraph() }; } diff --git a/src/core/graph/Graph.ts b/src/core/graph/Graph.ts index 5a37296d..b47d5259 100644 --- a/src/core/graph/Graph.ts +++ b/src/core/graph/Graph.ts @@ -9,7 +9,17 @@ export type { DeclaredPortKind } from '../properties/kinds'; * `picture` is what a shader's Pass draws (#301), for the nodes that read a picture. */ export type PortType = - 'exec' | 'number' | 'boolean' | 'string' | 'object' | 'list' | 'struct' | 'vec3' | 'picture' | 'any'; + | 'exec' + | 'number' + | 'boolean' + | 'string' + | 'object' + | 'list' + | 'struct' + | 'vec3' + | 'picture' + | 'audio' + | 'any'; export interface PortRef { node: string; @@ -172,11 +182,13 @@ export function graphFingerprint(graph: Graph): string { /** * A number may feed a colour: the shader compiler spreads it over the three channels. A picture is - * no value a calculation could take, so like `exec` it goes only where a picture is read. + * no value a calculation could take, so like `exec` it goes only where a picture is read – and a + * sound on its way through effects (#311) only where a sound goes in. */ export function portsCompatible(from: PortType, to: PortType): boolean { if (from === 'exec' || to === 'exec') return from === to; if (from === 'picture' || to === 'picture') return from === to; + if (from === 'audio' || to === 'audio') return from === to; if (from === 'number' && to === 'vec3') return true; return from === to || from === 'any' || to === 'any'; } diff --git a/src/core/graph/NodeContext.ts b/src/core/graph/NodeContext.ts index 1f818610..dd0e60f1 100644 --- a/src/core/graph/NodeContext.ts +++ b/src/core/graph/NodeContext.ts @@ -237,8 +237,9 @@ export interface NodeContext { * The voice belongs to the behavior that started it and ends when that behavior is destroyed – * a sound is something a thing in the world is doing. `detached` gives that up on purpose, for * the one case where the rule is in the way: the death sound of an object that destroys itself. + * `fx` names a sound FX asset the voice is played through (#311); `''` plays it as it is. */ - playSound(assetId: string, volume: number, loop: boolean | undefined, detached: boolean): void; + playSound(assetId: string, volume: number, loop: boolean | undefined, detached: boolean, fx: string): void; /** Silences every voice of that sound; `''` silences everything this run started. */ stopSound(assetId: string): void; /** diff --git a/src/core/graph/NodeRegistry.ts b/src/core/graph/NodeRegistry.ts index 93e5a4b6..7b96310b 100644 --- a/src/core/graph/NodeRegistry.ts +++ b/src/core/graph/NodeRegistry.ts @@ -30,11 +30,12 @@ export interface PortDef { } /** - * `state`, `shader` and `particles` are the categories no behavior graph offers: their nodes are the - * states of a state machine, the GLSL of a shader asset and the motion of one particle. + * `state`, `shader`, `particles` and `sound` are the categories no behavior graph offers: their nodes + * are the states of a state machine, the GLSL of a shader asset, the motion of one particle and the + * effects of a sound FX asset. */ export type NodeCategory = - 'event' | 'flow' | 'data' | 'variable' | 'action' | 'state' | 'shader' | 'particles'; + 'event' | 'flow' | 'data' | 'variable' | 'action' | 'state' | 'shader' | 'particles' | 'sound'; /** * How a loop node asks for one of its exec outputs to be run repeatedly: `port` carries the body, @@ -207,7 +208,13 @@ export function inlineInputs(def: NodeDef): PortDef[] { function isInlinePort(port: PortDef): boolean { if (port.inline === false) return false; if (port.type === 'vec3') return port.field !== undefined; - return port.type !== 'exec' && port.type !== 'object' && port.type !== 'struct' && port.type !== 'picture'; + return ( + port.type !== 'exec' && + port.type !== 'object' && + port.type !== 'struct' && + port.type !== 'picture' && + port.type !== 'audio' + ); } /** Untyped, an input writes the zero of its kind; as its default the node body shows it, not just the name. */ diff --git a/src/core/graph/nodes/action/PlaySound.ts b/src/core/graph/nodes/action/PlaySound.ts index 7de82622..44e8b1e8 100644 --- a/src/core/graph/nodes/action/PlaySound.ts +++ b/src/core/graph/nodes/action/PlaySound.ts @@ -1,6 +1,6 @@ import type { NodeDef, PortDef } from '../../NodeRegistry'; import { asBoolean, asNumber, asString, type Value } from '../../values'; -import { exec, num, soundPort, withDefault } from '../shared'; +import { exec, num, soundFxPort, soundPort, withDefault } from '../shared'; /** * Three answers, not two: a checkbox cannot say "leave it to the asset", and without that a sound @@ -56,14 +56,20 @@ export const PlaySound: NodeDef = { "leaves the asset's own setting alone or overrules it, so the asset says what the sound is and " + 'the node only colors it. Each play is a voice of its own: firing this twice in a row is heard ' + 'twice, overlapping. The sound belongs to this object and stops when the object is destroyed, ' + - 'unless "detached" is on.', - inputs: [exec(), soundPort(), withDefault(num('volume'), 1), loopPort(), detachedPort()], + 'unless "detached" is on. "fx" sends it through a sound FX asset; left empty, it plays as it is.', + inputs: [exec(), soundPort(), withDefault(num('volume'), 1), loopPort(), detachedPort(), soundFxPort()], outputs: [exec()], params: [], run: (ctx, inputs) => { const sound = asString(inputs.sound); if (sound) - ctx.playSound(sound, asNumber(inputs.volume), loopChoice(inputs.loop), asBoolean(inputs.detached)); + ctx.playSound( + sound, + asNumber(inputs.volume), + loopChoice(inputs.loop), + asBoolean(inputs.detached), + asString(inputs.fx), + ); return { next: 'exec' }; }, }; diff --git a/src/core/graph/nodes/index.ts b/src/core/graph/nodes/index.ts index 216b1ebe..03f5f97f 100644 --- a/src/core/graph/nodes/index.ts +++ b/src/core/graph/nodes/index.ts @@ -146,6 +146,7 @@ import { StateTransition } from './flow/StateTransition'; import { PARTICLE_NODES } from '../../particles/nodes'; import { SHADER_FUNCTION_NODES } from '../../shader/functions'; import { SHADER_NODES } from '../../shader/nodes'; +import { SOUND_FX_NODES } from '../../soundfx/nodes'; import { State } from '../stateMachine'; import { Variables } from './variable/Variables'; @@ -314,3 +315,4 @@ registerNode(State); for (const def of SHADER_NODES) registerNode(def); for (const def of SHADER_FUNCTION_NODES) registerNode(def); for (const def of PARTICLE_NODES) registerNode(def); +for (const def of SOUND_FX_NODES) registerNode(def); diff --git a/src/core/graph/nodes/shared.ts b/src/core/graph/nodes/shared.ts index 971a97dc..90cdeba3 100644 --- a/src/core/graph/nodes/shared.ts +++ b/src/core/graph/nodes/shared.ts @@ -94,6 +94,14 @@ export const soundPort = (): PortDef => ({ field: { key: 'sound', label: 'Sound', kind: 'sound', default: '' }, }); +/** The `fx` input of `Play sound` (#311); unconnected, it shows the picker of sound FX assets. */ +export const soundFxPort = (): PortDef => ({ + key: 'fx', + label: 'fx', + type: 'string', + field: { key: 'fx', label: 'FX', kind: 'asset', default: '', assetTypes: ['soundfx'] }, +}); + export function binaryMath(type: string, description: string, fn: (a: number, b: number) => number): NodeDef { return { type, diff --git a/src/core/model/Asset.ts b/src/core/model/Asset.ts index 15b53b19..fa36f344 100644 --- a/src/core/model/Asset.ts +++ b/src/core/model/Asset.ts @@ -310,6 +310,15 @@ export interface SoundAsset extends AssetBase { file: string; } +/** + * What a sound is played through (#311): the graph runs from its `Input` over effect nodes to its + * `Output`, and a `Play sound` naming it sends the voice along that line (see `core/soundfx`). + */ +export interface SoundFxAsset extends AssetBase { + type: 'soundfx'; + graph: Graph; +} + /** * A tile a tilemap paints with: `pictures` names one block per animation step (a texture id for a * sheet that holds nothing else, a frame id for one block cut out of a sheet), and `props` say @@ -562,6 +571,7 @@ export type Asset = | ShaderFunctionAsset | FontAsset | SoundAsset + | SoundFxAsset | ParticlesAsset; /** The assets whose `variables` are declared by the user – see `requireVariableOwner`. */ @@ -573,7 +583,7 @@ export function isVariableOwner(asset: Asset | undefined): asset is VariableOwne /** The assets edited as a node graph – what the graph canvas and the graph commands work on. */ export type GraphAsset = - BehaviorAsset | StateMachineAsset | ShaderAsset | ShaderFunctionAsset | ParticlesAsset; + BehaviorAsset | StateMachineAsset | ShaderAsset | ShaderFunctionAsset | ParticlesAsset | SoundFxAsset; const GRAPH_TYPES: ReadonlySet = new Set([ 'behavior', @@ -581,6 +591,7 @@ const GRAPH_TYPES: ReadonlySet = new Set([ 'shader', 'shaderfunction', 'particles', + 'soundfx', ]); export function isGraphAsset(asset: Asset): asset is GraphAsset { diff --git a/src/core/model/AssetTypes.ts b/src/core/model/AssetTypes.ts index 77082a1a..a333b4f5 100644 --- a/src/core/model/AssetTypes.ts +++ b/src/core/model/AssetTypes.ts @@ -24,6 +24,7 @@ export const ASSET_TYPES = [ 'shaderfunction', 'font', 'sound', + 'soundfx', 'particles', ] as const; @@ -55,6 +56,7 @@ export const ASSET_TYPE_LABELS: Record = { shaderfunction: 'Shader function', font: 'Font', sound: 'Sound', + soundfx: 'Sound FX', particles: 'Particles', }; @@ -66,7 +68,8 @@ export const ASSET_TYPE_LABELS: Record = { * what a tilemap draws with, so it is placed no more directly than a texture is – what is placed * is the tilemap. A curve is the way a key or a `Lerp` blends, picked by id like a state machine. * A struct is a shape that variables and fields take on (#253), nothing that exists on its own. - * A shader function is part of other shaders (#303), worn by nothing itself. + * A shader function is part of other shaders (#303), worn by nothing itself. A sound FX is what a + * sound is played through (#311), heard but never seen. */ export type PlaceableAssetType = Exclude< AssetType, @@ -83,6 +86,7 @@ export type PlaceableAssetType = Exclude< | 'shaderfunction' | 'font' | 'sound' + | 'soundfx' | 'variables' | 'struct' >; @@ -101,6 +105,7 @@ const NOT_PLACEABLE: ReadonlySet = new Set([ 'shaderfunction', 'font', 'sound', + 'soundfx', 'variables', 'struct', ]); diff --git a/src/core/properties/PropertySchema.ts b/src/core/properties/PropertySchema.ts index 089264a5..b756bf3d 100644 --- a/src/core/properties/PropertySchema.ts +++ b/src/core/properties/PropertySchema.ts @@ -655,6 +655,8 @@ const ASSET_SCHEMAS: Record = { { key: SOUND_VOLUME_KEY, label: 'Volume', kind: 'number', default: 1, min: 0, max: 1, step: 0.05 }, { key: SOUND_LOOP_KEY, label: 'Loop', kind: 'boolean', default: false }, ], + // The effects are the asset's graph (see SoundFxAsset), not a property. + soundfx: [], // How many particles and for how long; what one of them does is the asset's graph, whose Property // nodes add their rows after these – see `schemaForBag`. particles: [ diff --git a/src/core/soundfx/compile.ts b/src/core/soundfx/compile.ts new file mode 100644 index 00000000..0363d47d --- /dev/null +++ b/src/core/soundfx/compile.ts @@ -0,0 +1,67 @@ +import { graphEmitter } from '../graph/emitGraph'; +import { type Graph, graphFingerprint } from '../graph/Graph'; +import type { SoundFxAsset } from '../model/Asset'; +import { nodeDef } from '../graph/NodeRegistry'; +import { + DRY, + type FxChain, + isSoundFxNodeDef, + SOUND_FX_INPUT, + SOUND_FX_OUTPUT, + type SoundFxNodeDef, +} from './nodes'; + +export interface SoundFxBuild { + /** Null while the graph has a problem – a sound played through it is then played dry. */ + compiled: FxChain | null; + /** How many nodes between Input and Output the sound passes. */ + effects: number; + /** Per node id, what keeps the graph from compiling. */ + problems: Record; + /** One line for the tab: empty once compiled. */ + reason: string; +} + +/** + * A sound FX graph to the line a voice is sent along: walked back from the Output to the Input, every + * node on the way adding what it does. An input takes one edge, so the line never forks on its way + * to the Output; a node fed from nowhere is a problem rather than silence. + */ +export function compileSoundFx(graph: Graph): SoundFxBuild { + const walk = graphEmitter(graph, { + isDef: isSoundFxNodeDef, + noun: 'sound FX', + // Only reached with every input fed, so whatever is not the Input has its sound arriving. + emit: (def, inputs, node) => ({ out: def.emit(inputs.in ?? DRY, node.params) }), + unfed: (node) => { + walk.problem(node, 'Nothing comes in: connect it to the Input'); + return null; + }, + }); + + const output = graph.nodes.find((n) => n.type === SOUND_FX_OUTPUT); + if (!output) return { compiled: null, effects: 0, problems: walk.problems, reason: 'No Output node' }; + const chain = walk.input(output, nodeDef(SOUND_FX_OUTPUT)!.inputs[0]!); + const failed = walk.firstProblem(); + if (failed !== null || !chain) + return { compiled: null, effects: 0, problems: walk.problems, reason: failed ?? 'Cannot compile' }; + const effects = graph.nodes.filter( + (n) => walk.reached(n) && n.type !== SOUND_FX_INPUT && n.type !== SOUND_FX_OUTPUT, + ).length; + return { compiled: chain, effects, problems: walk.problems, reason: '' }; +} + +const builds = new Map(); + +/** + * The asset's build, compiled again only when what it does has changed – a voice is started far + * more often than its graph is edited, and a moved node changes nothing. + */ +export function soundFxBuildFor(asset: SoundFxAsset): SoundFxBuild { + const fingerprint = graphFingerprint(asset.graph); + const cached = builds.get(asset.id); + if (cached && cached.fingerprint === fingerprint) return cached.build; + const build = compileSoundFx(asset.graph); + builds.set(asset.id, { fingerprint, build }); + return build; +} diff --git a/src/core/soundfx/nodes.ts b/src/core/soundfx/nodes.ts new file mode 100644 index 00000000..d12287ec --- /dev/null +++ b/src/core/soundfx/nodes.ts @@ -0,0 +1,147 @@ +import type { NodeDef, PortDef } from '../graph/NodeRegistry'; +import type { PropertyBag } from '../model/Properties'; +import type { FlatDef } from '../properties/PropertySchema'; + +/** The filter shapes the EQ uses – Web Audio's names, so the player hands them on as they are. */ +export type FxFilter = 'lowshelf' | 'peaking' | 'highshelf'; + +/** One step a voice passes on its way out: a volume, or one band of an equalizer. */ +export type FxStage = + | { kind: 'gain'; gain: number } + | { kind: 'filter'; filter: FxFilter; frequency: number; gain: number; q: number }; + +/** + * A sound on its way through the graph: how many semitones it is played up or down, and what it + * passes, in order. `gain` is a factor, a filter's `gain` is in dB as Web Audio takes it. + */ +export interface FxChain { + pitch: number; + stages: readonly FxStage[]; +} + +export const DRY: FxChain = { pitch: 0, stages: [] }; + +/** + * A node of a sound FX graph (#311). It never runs: `emit` says what it does to the sound arriving + * at its one input. Input and Output have a side less and no effect of their own. + */ +export interface SoundFxNodeDef extends NodeDef { + category: 'sound'; + emit(chain: FxChain, params: PropertyBag): FxChain; +} + +export function isSoundFxNodeDef(def: NodeDef): def is SoundFxNodeDef { + return def.category === 'sound' && 'emit' in def; +} + +export const SOUND_FX_INPUT = 'SoundFxInput'; +export const SOUND_FX_OUTPUT = 'SoundFxOutput'; + +const audioIn: PortDef = { key: 'in', label: 'in', type: 'audio' }; +const audioOut: PortDef = { key: 'out', label: 'out', type: 'audio' }; + +function numberParam(key: string, label: string, min: number, max: number, step: number): FlatDef<'number'> { + return { key, label, kind: 'number', default: 0, min, max, step }; +} + +/** A param's value held to its row, so a hand-edited project cannot blow a speaker. */ +function read(params: PropertyBag, def: FlatDef<'number'>): number { + const value = Number(params[def.key] ?? def.default); + if (!Number.isFinite(value)) return def.default; + return Math.max(def.min ?? -Infinity, Math.min(def.max ?? Infinity, value)); +} + +export const SoundFxInput: SoundFxNodeDef = { + type: SOUND_FX_INPUT, + label: 'Input', + category: 'sound', + description: 'The sound as the Play sound node starts it, before any effect.', + inputs: [], + outputs: [audioOut], + params: [], + emit: (chain) => chain, +}; + +export const SoundFxOutput: SoundFxNodeDef = { + type: SOUND_FX_OUTPUT, + label: 'Output', + category: 'sound', + description: 'What is heard: the sound as it arrives here, with every effect on the way.', + inputs: [audioIn], + outputs: [], + params: [], + emit: (chain) => chain, +}; + +const GAIN = numberParam('gain', 'Gain (dB)', -60, 24, 0.5); + +export const SoundFxGain: SoundFxNodeDef = { + type: 'SoundFxGain', + label: 'Gain', + category: 'sound', + description: + 'Makes the sound louder or quieter, in decibels: 0 leaves it as it is, -6 is about half as loud, +6 twice. Above 0 it can clip.', + inputs: [audioIn], + outputs: [audioOut], + params: [GAIN], + emit: (chain, params) => { + const db = read(params, GAIN); + if (db === 0) return chain; + return { ...chain, stages: [...chain.stages, { kind: 'gain', gain: 10 ** (db / 20) }] }; + }, +}; + +const SEMITONES = numberParam('semitones', 'Semitones', -24, 24, 1); + +export const SoundFxPitch: SoundFxNodeDef = { + type: 'SoundFxPitch', + label: 'Pitch', + category: 'sound', + description: + 'Plays the sound higher or lower, in semitones: 12 is an octave up. It is the speed the sound is played at, so a higher one is also shorter, and it applies to the whole sound wherever the node sits in the line.', + inputs: [audioIn], + outputs: [audioOut], + params: [SEMITONES], + emit: (chain, params) => ({ ...chain, pitch: chain.pitch + read(params, SEMITONES) }), +}; + +/** The five bands of the EQ: a shelf at either end and three peaks between them. */ +export const EQ_BANDS: readonly { key: string; label: string; filter: FxFilter; frequency: number }[] = [ + { key: 'low', label: '80 Hz', filter: 'lowshelf', frequency: 80 }, + { key: 'lowMid', label: '250 Hz', filter: 'peaking', frequency: 250 }, + { key: 'mid', label: '1 kHz', filter: 'peaking', frequency: 1000 }, + { key: 'highMid', label: '4 kHz', filter: 'peaking', frequency: 4000 }, + { key: 'high', label: '12 kHz', filter: 'highshelf', frequency: 12000 }, +]; + +/** About an octave and a half wide: neighbouring peaks overlap, so no frequency falls between them. */ +const PEAK_Q = 1; +const EQ_PARAMS = EQ_BANDS.map((band) => numberParam(band.key, band.label, -24, 24, 0.5)); + +export const SoundFxEq: SoundFxNodeDef = { + type: 'SoundFxEq', + label: 'EQ', + category: 'sound', + description: + 'A five-band equalizer: raises or lowers the lows (80 Hz), low mids (250 Hz), mids (1 kHz), high mids (4 kHz) and highs (12 kHz), each in decibels. A band at 0 does nothing.', + inputs: [audioIn], + outputs: [audioOut], + params: EQ_PARAMS, + emit: (chain, params) => { + const stages: FxStage[] = []; + EQ_BANDS.forEach((band, i) => { + const gain = read(params, EQ_PARAMS[i]!); + if (gain !== 0) + stages.push({ kind: 'filter', filter: band.filter, frequency: band.frequency, gain, q: PEAK_Q }); + }); + return stages.length === 0 ? chain : { ...chain, stages: [...chain.stages, ...stages] }; + }, +}; + +export const SOUND_FX_NODES: readonly SoundFxNodeDef[] = [ + SoundFxInput, + SoundFxOutput, + SoundFxGain, + SoundFxPitch, + SoundFxEq, +]; diff --git a/src/core/store/serialize.ts b/src/core/store/serialize.ts index 9dbfac39..3c0d917a 100644 --- a/src/core/store/serialize.ts +++ b/src/core/store/serialize.ts @@ -19,6 +19,7 @@ import { } from '../graph'; import { isParticleNodeDef } from '../particles/nodes'; import { isShaderNodeDef } from '../shader/nodes'; +import { isSoundFxNodeDef } from '../soundfx/nodes'; import { isKeyInterpolation, normalizeKeys } from '../model/animation'; import type { AnimationKey, @@ -234,6 +235,8 @@ export function deserializeProject(raw: unknown, fallbackId = ''): Project { assets.push({ id, type, name, props, graph: shaderGraphs.get(id) ?? createEmptyGraph() }); } else if (type === 'particles') { assets.push({ id, type, name, props, graph: particleGraphs.get(id) ?? createEmptyGraph() }); + } else if (type === 'soundfx') { + assets.push({ id, type, name, props, graph: normalizeGraph(a.graph, isSoundFxNodeDef) }); } else if (type === 'texture') { assets.push({ id, type, name, props, ...normalizeTexture(a, takenIds) }); } else if (type === 'font') { diff --git a/src/editor/Editor.ts b/src/editor/Editor.ts index 957f8b1c..1762ed03 100644 --- a/src/editor/Editor.ts +++ b/src/editor/Editor.ts @@ -72,6 +72,7 @@ import { type PlayHost, PlaySession, type WorldSnapshot } from './PlaySession'; import { ProjectSession } from './ProjectSession'; import { ParticlesEditor } from './particles/ParticlesEditor'; import { ShaderEditor } from './shader/ShaderEditor'; +import { SoundFxEditor } from './soundfx/SoundFxEditor'; import { StateMachineEditor } from './statemachine/StateMachineEditor'; import { installShortcuts } from './shortcuts'; import type { PlayFrom } from '../core/model/playSettings'; @@ -93,6 +94,7 @@ export type CentreTab = | 'statemachine' | 'shader' | 'particles' + | 'soundfx' | 'game' | 'texture' | 'vectorart' @@ -142,6 +144,7 @@ const PANE_ORDER = [ 'statemachine', 'shader', 'particles', + 'soundfx', 'texture', 'vectorart', 'pixelart', @@ -161,6 +164,7 @@ const TAB_LABELS: Record = { statemachine: 'State machine', shader: 'Shader', particles: 'Particles', + soundfx: 'Sound FX', texture: 'Texture', vectorart: 'Vector art', pixelart: 'Pixel art', @@ -209,6 +213,7 @@ export class Editor implements PlayHost { readonly stateMachineEditor: StateMachineEditor; readonly shaderEditor: ShaderEditor; readonly particlesEditor: ParticlesEditor; + readonly soundFxEditor: SoundFxEditor; readonly textureEditor: TextureEditor; readonly vectorArtEditor: VectorArtEditor; readonly pixelArtEditor: PixelArtEditor; @@ -294,6 +299,7 @@ export class Editor implements PlayHost { this.stateMachineEditor = new StateMachineEditor(state); this.shaderEditor = new ShaderEditor(state); this.particlesEditor = new ParticlesEditor(state); + this.soundFxEditor = new SoundFxEditor(state); this.textureEditor = new TextureEditor(state); this.vectorArtEditor = new VectorArtEditor(state); this.pixelArtEditor = new PixelArtEditor(state); @@ -402,6 +408,7 @@ export class Editor implements PlayHost { statemachine: this.stateMachineEditor, shader: this.shaderEditor, particles: this.particlesEditor, + soundfx: this.soundFxEditor, texture: this.textureEditor, vectorart: this.vectorArtEditor, pixelart: this.pixelArtEditor, @@ -591,6 +598,8 @@ export class Editor implements PlayHost { return this.shaderEditor.canvas.paste(); case 'particles': return this.particlesEditor.canvas.paste(); + case 'soundfx': + return this.soundFxEditor.canvas.paste(); default: return false; } diff --git a/src/editor/assetTabs.ts b/src/editor/assetTabs.ts index ce770451..e10936e8 100644 --- a/src/editor/assetTabs.ts +++ b/src/editor/assetTabs.ts @@ -9,6 +9,7 @@ export const TAB_OF_ASSET: Partial> = { shader: 'shader', shaderfunction: 'shader', particles: 'particles', + soundfx: 'soundfx', texture: 'texture', vectorart: 'vectorart', pixelart: 'pixelart', diff --git a/src/editor/debug/graphHooks.ts b/src/editor/debug/graphHooks.ts index e5f267e7..a99b9e31 100644 --- a/src/editor/debug/graphHooks.ts +++ b/src/editor/debug/graphHooks.ts @@ -96,6 +96,20 @@ export function graphHooks({ state, editor }: HookContext) { startRing: (nodeId: string) => editor.stateMachineEditor.canvas.startRingDrawn(nodeId), stateMarks: () => editor.stateMachineEditor.canvas.stateMarksDrawn(), }, + soundfx: { + current: () => editor.soundFxEditor.currentAssetId, + info: () => editor.soundFxEditor.shownInfo, + canvasRect: () => editor.soundFxEditor.canvas.canvasRect(), + nodeProblem: (nodeId: string) => editor.soundFxEditor.canvas.nodeProblem(nodeId), + /** The header's listening controls: which sound, and whether Play is offered. */ + preview: () => ({ + sound: editor.soundFxEditor.previewSoundId, + picker: editor.soundFxEditor.soundPicker.text, + play: editor.soundFxEditor.playButton.enabled, + }), + playAt: () => centreOf(editor.soundFxEditor.playButton), + stopAt: () => centreOf(editor.soundFxEditor.stopButton), + }, particles: { current: () => editor.particlesEditor.currentAssetId, info: () => editor.particlesEditor.shownInfo, diff --git a/src/editor/graph/GraphCanvas.ts b/src/editor/graph/GraphCanvas.ts index c3df36f1..960eb8e0 100644 --- a/src/editor/graph/GraphCanvas.ts +++ b/src/editor/graph/GraphCanvas.ts @@ -28,6 +28,7 @@ import type { GraphAsset } from '../../core/model/Asset'; import { graphLookup } from '../../core/model/graphLookup'; import { particleLibrary } from '../../viewport/ParticleLibrary'; import { shaderLibrary } from '../../viewport/ShaderLibrary'; +import { soundFxBuildFor } from '../../core/soundfx/compile'; import { findAsset, findGraphAsset } from '../../core/model/Project'; import { type Trace, TRACE_FADE } from '../../runtime/systems/BehaviorSystem'; import { ViewTween } from '../viewTween'; @@ -305,7 +306,9 @@ export class GraphCanvas extends Widget { ? shaderLibrary.checkFor(asset, this.state.project).problems : asset.type === 'particles' ? particleLibrary.buildFor(asset).problems - : null; + : asset.type === 'soundfx' + ? soundFxBuildFor(asset).problems + : null; for (const node of asset.graph.nodes) { const resolved = resolveDef(node, asset.graph, lookup); if (!resolved) continue; diff --git a/src/editor/graph/NodeView.ts b/src/editor/graph/NodeView.ts index d70554bf..38d6760a 100644 --- a/src/editor/graph/NodeView.ts +++ b/src/editor/graph/NodeView.ts @@ -67,6 +67,8 @@ function categoryColor(category: NodeCategory): number { return c.categoryShader; case 'particles': return c.categoryParticles; + case 'sound': + return c.categorySound; } } @@ -91,6 +93,8 @@ export function portColor(type: PortType): number { return c.portVec3; case 'picture': return c.portPicture; + case 'audio': + return c.portAudio; case 'any': return c.portAny; } diff --git a/src/editor/graph/paletteEntries.ts b/src/editor/graph/paletteEntries.ts index 934fa413..2a039ba7 100644 --- a/src/editor/graph/paletteEntries.ts +++ b/src/editor/graph/paletteEntries.ts @@ -41,6 +41,7 @@ import { import { nextParticlePropertyName, PARTICLE_NODES, PARTICLE_OUTPUT } from '../../core/particles/nodes'; import { offeredIn, ShaderFunctionInput, ShaderFunctionOutput } from '../../core/shader/functions'; import { SHADER_NODES, SHADER_OUTPUT, SHADER_VERTEX_OUTPUT } from '../../core/shader/nodes'; +import { SOUND_FX_INPUT, SOUND_FX_NODES, SOUND_FX_OUTPUT } from '../../core/soundfx/nodes'; /** * One row of the palette. `def` is the definition the node would actually have once placed, so the @@ -125,7 +126,8 @@ function shaderEntries(project: Project, asset: ShaderAsset | ShaderFunctionAsse /** * The node types offerable in `asset`'s graph. A state machine holds states and nothing else, each - * new one named by its number; a shader holds shader nodes (`shaderEntries`). In a behavior, every other behavior appears under its own name instead of behind a + * new one named by its number; a shader holds shader nodes (`shaderEntries`), a sound FX its effects + * with one Input and one Output. In a behavior, every other behavior appears under its own name instead of behind a * generic "Run behavior" – and once more as the event "On " if it has an End to be heard * through (#294) –, and the two terminals drop out once the graph has them. */ @@ -135,6 +137,12 @@ export function paletteEntries(project: Project, asset: GraphAsset): PaletteEntr return [{ id: State.type, hint: State.category, def: State, params }]; } if (asset.type === 'shader' || asset.type === 'shaderfunction') return shaderEntries(project, asset); + if (asset.type === 'soundfx') { + const placed = new Set(asset.graph.nodes.map((n) => n.type)); + return SOUND_FX_NODES.filter( + (def) => !(def.type === SOUND_FX_INPUT || def.type === SOUND_FX_OUTPUT) || !placed.has(def.type), + ).map((def) => ({ id: def.type, hint: def.category, def, params: {} })); + } if (asset.type === 'particles') { const hasOutput = asset.graph.nodes.some((n) => n.type === PARTICLE_OUTPUT); const taken = new Set(asset.graph.nodes.filter(isParticlePropertyNode).map(propertyNameOf)); @@ -147,7 +155,7 @@ export function paletteEntries(project: Project, asset: GraphAsset): PaletteEntr const entries: PaletteEntry[] = []; for (const def of allNodeDefs()) { if (namesBehavior(def.type) || def.category === 'state') continue; - if (def.category === 'shader' || def.category === 'particles') continue; + if (def.category === 'shader' || def.category === 'particles' || def.category === 'sound') continue; if (isTerminalType(def.type) && terminalNode(asset.graph, def.type)) continue; entries.push({ id: def.type, hint: def.category, def, params: {} }); if (def.type === FIND) entries.push(filterEntry(def)); diff --git a/src/editor/panels/assetGroups.ts b/src/editor/panels/assetGroups.ts index 901e961d..3fc33a7f 100644 --- a/src/editor/panels/assetGroups.ts +++ b/src/editor/panels/assetGroups.ts @@ -34,6 +34,8 @@ export const ASSET_GROUP_LABELS: Record = { shaderfunction: 'Shader functions', font: 'Fonts', sound: 'Sounds', + // Not "Sound FX": that is the `+` menu entry (see Vector art). + soundfx: 'Sound effects', particles: 'Particles', }; diff --git a/src/editor/panels/assetIcons.ts b/src/editor/panels/assetIcons.ts index 7df18c38..4c33ef43 100644 --- a/src/editor/panels/assetIcons.ts +++ b/src/editor/panels/assetIcons.ts @@ -24,6 +24,7 @@ import scene from './icons/scene.svg?raw'; import shader from './icons/shader.svg?raw'; import shaderfunction from './icons/shaderfunction.svg?raw'; import sound from './icons/sound.svg?raw'; +import soundfx from './icons/soundfx.svg?raw'; import sprite from './icons/sprite.svg?raw'; import statemachine from './icons/statemachine.svg?raw'; import struct from './icons/struct.svg?raw'; @@ -53,6 +54,7 @@ export const ASSET_ICON_SVG: Record = { shader, shaderfunction, sound, + soundfx, sprite, statemachine, struct, diff --git a/src/editor/panels/icons/soundfx.svg b/src/editor/panels/icons/soundfx.svg new file mode 100644 index 00000000..ccf04480 --- /dev/null +++ b/src/editor/panels/icons/soundfx.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/editor/soundfx/SoundFxEditor.ts b/src/editor/soundfx/SoundFxEditor.ts new file mode 100644 index 00000000..e1ee326a --- /dev/null +++ b/src/editor/soundfx/SoundFxEditor.ts @@ -0,0 +1,107 @@ +import type { Asset, SoundFxAsset } from '../../core/model/Asset'; +import { findAssetOf } from '../../core/model/Project'; +import { resolveAssetProps } from '../../core/properties/resolve'; +import { volumeOf } from '../../core/properties/sounds'; +import { soundFxBuildFor } from '../../core/soundfx/compile'; +import { Button, Dropdown, Menu } from '../../ui'; +import { soundLibrary } from '../../viewport/SoundLibrary'; +import { AssetTabEditor } from '../AssetTabEditor'; +import { GraphCanvas } from '../graph/GraphCanvas'; +import { SnapToggle } from '../graph/SnapToggle'; +import { TOOLBAR_ICON_SVG } from '../panels/toolbarIcons'; +import type { EditorState } from '../state'; + +/** Room kept for "N effects", or for the reason the graph does not compile. */ +const INFO_WIDTH = 220; +const SOUND_WIDTH = 140; +/** Whose voices the tab's Stop silences: its own, never a running game's. */ +const PREVIEW_OWNER = 'soundfx-preview'; + +/** + * The nodes of one sound FX asset (#311), and in the header a way to hear them: a sound of the + * project played through the graph as it stands. Which sound is the tab's, not the asset's – it is + * only there to listen with. + */ +export class SoundFxEditor extends AssetTabEditor { + readonly snap = new SnapToggle(() => this.canvas); + /** Public for the check scripts. */ + readonly soundPicker: Dropdown; + readonly playButton: Button; + readonly stopButton: Button; + private soundId = ''; + + constructor(state: EditorState) { + super(state, { infoWidth: INFO_WIDTH }); + this.soundPicker = new Dropdown('No sound', { + width: SOUND_WIDTH, + onClick: () => this.openSoundPicker(), + }); + this.playButton = new Button('Play', { icon: TOOLBAR_ICON_SVG.play, onClick: () => this.play() }); + this.stopButton = new Button('Stop', { icon: TOOLBAR_ICON_SVG.stop, onClick: () => this.stop() }); + this.mount(new GraphCanvas(state, 'sound FX'), [ + ...this.snap.widgets, + this.soundPicker, + this.playButton, + this.stopButton, + ]); + state.on('load', () => { + this.soundId = ''; + }); + } + + override selectedId(): string { + return this.state.selectedSoundFxId ?? ''; + } + + override isMine(asset: Asset): asset is SoundFxAsset { + return asset.type === 'soundfx'; + } + + /** The sound the header plays: the one picked, else the project's first. */ + get previewSoundId(): string { + const sounds = this.state.project.assets.filter((a) => a.type === 'sound'); + return sounds.some((a) => a.id === this.soundId) ? this.soundId : (sounds[0]?.id ?? ''); + } + + pickSound(id: string): void { + this.soundId = id; + this.refresh(); + } + + /** Plays the picked sound through the graph; a graph that does not compile is not played at all. */ + play(): void { + const asset = this.asset(); + const sound = findAssetOf(this.state.project, this.previewSoundId, 'sound'); + const chain = asset && soundFxBuildFor(asset).compiled; + if (!sound?.file || !chain) return; + const volume = volumeOf(resolveAssetProps(this.state.project, sound)); + soundLibrary.play(sound.file, volume, false, PREVIEW_OWNER, chain); + } + + stop(): void { + soundLibrary.stopOwner(PREVIEW_OWNER); + } + + protected override infoText(asset: SoundFxAsset): string { + const build = soundFxBuildFor(asset); + if (!build.compiled) return build.reason; + return `${build.effects} ${build.effects === 1 ? 'effect' : 'effects'}`; + } + + protected override syncExtras(asset: SoundFxAsset | null): void { + this.snap.setEnabled(Boolean(asset)); + const sound = findAssetOf(this.state.project, this.previewSoundId, 'sound'); + this.soundPicker.setText(sound?.name ?? 'No sound'); + this.soundPicker.setEnabled(Boolean(asset && sound)); + this.playButton.setEnabled(Boolean(asset && sound?.file && soundFxBuildFor(asset).compiled)); + this.stopButton.setEnabled(Boolean(asset)); + } + + private openSoundPicker(): void { + const current = this.previewSoundId; + const items = this.state.project.assets + .filter((a) => a.type === 'sound') + .map((a) => ({ label: a.name, current: a.id === current, onSelect: () => this.pickSound(a.id) })); + if (items.length > 0) new Menu(items).openBelow(this.soundPicker); + } +} diff --git a/src/editor/state.ts b/src/editor/state.ts index e2b793dc..40eecea5 100644 --- a/src/editor/state.ts +++ b/src/editor/state.ts @@ -810,6 +810,11 @@ export class EditorState { return this.selectedGraphId('particles'); } + /** The sound FX asset the selection refers to: the asset, or one of its nodes. */ + get selectedSoundFxId(): string | null { + return this.selectedGraphId('soundfx'); + } + private selectedGraphId(type: GraphAsset['type']): string | null { const s = this.selection; if (!s) return null; diff --git a/src/runtime/systems/Audio.ts b/src/runtime/systems/Audio.ts index f9d6d5c4..b2977a0a 100644 --- a/src/runtime/systems/Audio.ts +++ b/src/runtime/systems/Audio.ts @@ -1,3 +1,5 @@ +import type { FxChain } from '../../core/soundfx/nodes'; + /** * What the runtime needs of the outside world to make a noise – the counterpart of `Input` on the * way out. The browser side of it is `viewport/SoundLibrary`; a test (and a run started before @@ -11,9 +13,10 @@ export interface AudioSink { * Starts one voice. `volume` is final (0–1) and every call overlaps whatever is already playing. * `owner` is the behavior the sound belongs to: when that behavior is destroyed the voice goes * with it, because a sound is something a thing in the world is doing, not a thing of its own. - * `''` means the voice has no owner and outlives whatever started it. + * `''` means the voice has no owner and outlives whatever started it. `fx` is the line of effects + * the voice passes on its way out (#311); `DRY` passes none. */ - play(file: string, volume: number, loop: boolean, owner: string): void; + play(file: string, volume: number, loop: boolean, owner: string, fx?: FxChain): void; /** Silences every voice of that file; `''` silences all of them. */ stop(file: string): void; /** Silences every voice started by that owner – what destroying a behavior does. */ diff --git a/src/runtime/systems/behaviorContext.ts b/src/runtime/systems/behaviorContext.ts index 1e2be81d..0abcb034 100644 --- a/src/runtime/systems/behaviorContext.ts +++ b/src/runtime/systems/behaviorContext.ts @@ -15,6 +15,8 @@ import { soundFileOf } from '../../core/model/sound'; import { hasRenderLayer } from '../../core/properties/PropertySchema'; import { resolveAssetProps } from '../../core/properties/resolve'; import { loopsOf, volumeOf } from '../../core/properties/sounds'; +import { soundFxBuildFor } from '../../core/soundfx/compile'; +import { DRY } from '../../core/soundfx/nodes'; import type { Entity } from '../Entity'; import { PhysicsSystem } from '../physics/PhysicsSystem'; import type { World } from '../World'; @@ -162,8 +164,8 @@ export function behaviorContext( // eslint-disable-next-line no-console console.log(`[behavior ${entity.name}]`, message); }, - playSound: (assetId, volume, loop, detached) => - playSound(run, assetId, volume, loop, detached ? '' : entity.id), + playSound: (assetId, volume, loop, detached, fx) => + playSound(run, assetId, volume, loop, detached ? '' : entity.id, fx), // An id nothing resolves stops nothing, rather than silencing the whole run by accident; // only the empty id means "everything", which is what the node promises. stopSound: (assetId) => { @@ -214,7 +216,8 @@ export function behaviorContext( /** * What the graph asked for, as the sink understands it: the asset's own volume scaled by the * node's, and its loop flag unless the node overrules it either way. The asset is what a sound - * *is* – a node that says nothing therefore plays it as the project describes it. + * *is* – a node that says nothing therefore plays it as the project describes it. An FX asset that + * does not compile plays the sound dry: a broken effect should not silence the game. */ function playSound( { project, audio }: ContextSources, @@ -222,12 +225,14 @@ function playSound( volume: number, loop: boolean | undefined, owner: string, + fxId: string, ): void { const asset = findAssetOf(project, assetId, 'sound'); if (!asset?.file) return; const props = resolveAssetProps(project, asset); const level = Math.max(0, Math.min(1, volumeOf(props) * volume)); - audio.play(asset.file, level, loop ?? loopsOf(props), owner); + const fx = fxId ? findAssetOf(project, fxId, 'soundfx') : undefined; + audio.play(asset.file, level, loop ?? loopsOf(props), owner, (fx && soundFxBuildFor(fx).compiled) || DRY); } /** Where a ray of that length ends; where it has no direction, where it starts. */ diff --git a/src/ui/Theme.ts b/src/ui/Theme.ts index 4af1fe95..9d0a2580 100644 --- a/src/ui/Theme.ts +++ b/src/ui/Theme.ts @@ -69,6 +69,7 @@ export const Theme = { categoryState: 0x8e5ad6, categoryShader: 0x1f8f8f, categoryParticles: 0xc26a1f, + categorySound: 0xb0457a, portExec: 0xffffff, portNumber: 0x6fcf97, portBoolean: 0xe57373, @@ -78,6 +79,7 @@ export const Theme = { portStruct: 0xd4c25a, portVec3: 0xff9ad5, portPicture: 0x9b8cff, + portAudio: 0xf0c060, portAny: 0xbdbdbd, /** Ring around the port a running edge drag would connect to. */ portTarget: 0xffffff, diff --git a/src/viewport/SoundLibrary.ts b/src/viewport/SoundLibrary.ts index df1a28d8..307708bf 100644 --- a/src/viewport/SoundLibrary.ts +++ b/src/viewport/SoundLibrary.ts @@ -1,3 +1,4 @@ +import { DRY, type FxChain, type FxStage } from '../core/soundfx/nodes'; import type { AudioSink } from '../runtime/systems/Audio'; const DEFAULT_BASE = '/game-maker/data/sounds'; @@ -35,7 +36,7 @@ export class SoundLibrary implements AudioSink { private started = 0; private stopped = 0; /** The last voice actually started, with the values it was started at – what a check reads back. */ - private last: { file: string; volume: number; loop: boolean } | null = null; + private last: LastVoice | null = null; /** * How often a stop has reached each file, and how often one has silenced everything. Counters * rather than a "cancelled" flag: what a play has to know is whether a stop happened *after it @@ -53,18 +54,19 @@ export class SoundLibrary implements AudioSink { this.base = base.replace(/\/+$/, ''); } - play(file: string, volume: number, loop: boolean, owner = ''): void { + play(file: string, volume: number, loop: boolean, owner = '', fx: FxChain = DRY): void { if (file === '' || this.failed.has(file)) return; const buffer = this.buffers.get(file); if (buffer) { - this.start(file, buffer, volume, loop, owner); + this.start(file, buffer, volume, loop, owner, fx); return; } const askedAt = this.cancelsFor(file, owner); void this.load(file).then((loaded) => { // A `Stop sound` – or the destruction of the behavior that asked – that happened while the // bytes were in flight has to win, or a sound the run already ended would start afterwards. - if (loaded && this.cancelsFor(file, owner) === askedAt) this.start(file, loaded, volume, loop, owner); + if (loaded && this.cancelsFor(file, owner) === askedAt) + this.start(file, loaded, volume, loop, owner, fx); }); } @@ -104,7 +106,7 @@ export class SoundLibrary implements AudioSink { stopped: number; decoded: number; failed: number; - last: { file: string; volume: number; loop: boolean } | null; + last: LastVoice | null; } { return { playing: this.voices.length, @@ -155,7 +157,14 @@ export class SoundLibrary implements AudioSink { this.stopped += going.length; } - private start(file: string, buffer: AudioBuffer, volume: number, loop: boolean, owner: string): void { + private start( + file: string, + buffer: AudioBuffer, + volume: number, + loop: boolean, + owner: string, + fx: FxChain, + ): void { const context = this.audio(); // A context starts suspended where the page has not been interacted with; without this the // voice would run silently and end as if it had played. @@ -163,10 +172,13 @@ export class SoundLibrary implements AudioSink { const source = context.createBufferSource(); source.buffer = buffer; source.loop = loop; + source.playbackRate.value = 2 ** (fx.pitch / 12); const gain = context.createGain(); gain.gain.value = Math.max(0, Math.min(1, volume)); - this.last = { file, volume: gain.gain.value, loop }; - source.connect(gain).connect(context.destination); + this.last = { file, volume: gain.gain.value, loop, pitch: fx.pitch, stages: fx.stages.length }; + let tail: AudioNode = source; + for (const stage of fx.stages) tail = tail.connect(fxNode(context, stage)); + tail.connect(gain).connect(context.destination); const voice: Voice = { file, owner, source, gain }; source.onended = () => { this.voices = this.voices.filter((other) => other !== voice); @@ -186,4 +198,28 @@ export class SoundLibrary implements AudioSink { } } +/** What `stats` says about the last voice: enough for a check to tell which effects it went through. */ +interface LastVoice { + file: string; + volume: number; + loop: boolean; + /** Semitones, and how many gain and filter nodes it passed. */ + pitch: number; + stages: number; +} + +function fxNode(context: AudioContext, stage: FxStage): AudioNode { + if (stage.kind === 'gain') { + const gain = context.createGain(); + gain.gain.value = stage.gain; + return gain; + } + const filter = context.createBiquadFilter(); + filter.type = stage.filter; + filter.frequency.value = stage.frequency; + filter.gain.value = stage.gain; + filter.Q.value = stage.q; + return filter; +} + export const soundLibrary = new SoundLibrary(); diff --git a/tests/core/graph.test.ts b/tests/core/graph.test.ts index 909c9edd..f6c296f4 100644 --- a/tests/core/graph.test.ts +++ b/tests/core/graph.test.ts @@ -79,8 +79,10 @@ describe('node registry', () => { if (def.category === 'event') expect(def.inputs.filter((p) => p.type === 'exec')).toHaveLength(0); // A state is a name in a machine; its ports only anchor transitions, so it never runs. else if (def.category === 'state') expect(def.run).toBeUndefined(); - // A shader node is compiled to GLSL and a particle node to a function: neither is ever run. - else if (def.category === 'shader' || def.category === 'particles') expect('emit' in def).toBe(true); + // A shader node is compiled to GLSL, a particle node to a function and a sound FX node to a + // stage of a voice: none is ever run. + else if (def.category === 'shader' || def.category === 'particles' || def.category === 'sound') + expect('emit' in def).toBe(true); // The end of a block runs as part of the node that opens it (`Find` … `Match`). else if (defs.some((d) => d.opens?.type === def.type)) expect(def.run).toBeUndefined(); // Everything else does one of the two: a step (`run`), or an answer whenever asked (`read`). diff --git a/tests/core/node-catalog.test.ts b/tests/core/node-catalog.test.ts index 9a758569..845b36a8 100644 --- a/tests/core/node-catalog.test.ts +++ b/tests/core/node-catalog.test.ts @@ -1950,7 +1950,7 @@ const CASES: Record = { // flag – is the BehaviorSystem's to resolve, since only it holds the project (see // `sounds.test.ts`). after: (ctx) => - expect(ctx.played).toEqual([{ assetId: 'snd-1', volume: 0.5, loop: true, detached: false }]), + expect(ctx.played).toEqual([{ assetId: 'snd-1', volume: 0.5, loop: true, detached: false, fx: '' }]), }, { name: 'detached says the voice is nobody’s, so destroying this object leaves it playing', @@ -1970,6 +1970,12 @@ const CASES: Record = { next: 'exec', after: (ctx) => expect(ctx.played[0]?.loop).toBe(false), }, + { + name: 'hands on the sound FX it names (#311)', + inputs: { sound: 'snd-1', volume: 1, loop: '', fx: 'fx-1' }, + next: 'exec', + after: (ctx) => expect(ctx.played[0]?.fx).toBe('fx-1'), + }, ], LoadScene: [ { diff --git a/tests/core/soundfx.test.ts b/tests/core/soundfx.test.ts new file mode 100644 index 00000000..f84ac396 --- /dev/null +++ b/tests/core/soundfx.test.ts @@ -0,0 +1,219 @@ +/** + * A sound FX asset (#311) is a graph a sound is played through: Input, effects, Output. It never + * runs as a behavior does – the compiler turns it into the line of effects a voice passes. + */ +import { describe, expect, it } from 'vitest'; +import { buildAsset, CreateAssetCommand, CreateSoundCommand } from '../../src/core/commands/assetCommands'; +import { portsCompatible } from '../../src/core/graph/Graph'; +import type { BehaviorAsset, SoundFxAsset } from '../../src/core/model/Asset'; +import { isPlaceableType } from '../../src/core/model/AssetTypes'; +import { assetRefChoices } from '../../src/core/properties/assetRef'; +import { compileSoundFx, soundFxBuildFor } from '../../src/core/soundfx/compile'; +import { DRY, type FxChain } from '../../src/core/soundfx/nodes'; +import { deserializeProject } from '../../src/core/store/serialize'; +import type { AudioSink } from '../../src/runtime/systems/Audio'; +import { paletteEntries } from '../../src/editor/graph/paletteEntries'; +import { + graphEditor, + makeBehavior, + makeProject, + makeSpriteScene, + place, + type ProjectFixture, + roundTrip, + startWorld, +} from '../helpers'; + +const FILE = `${'a1'.repeat(32)}.wav`; + +function makeFx(f: ProjectFixture = makeProject()) { + const command = new CreateAssetCommand('soundfx', 'Cave'); + f.stack.execute(command); + const fx = command.asset as SoundFxAsset; + return { ...f, fx, edit: graphEditor(f, fx as unknown as BehaviorAsset) }; +} + +/** Input, the given effects in a row, Output. */ +function line(effects: { type: string; params?: Record }[]) { + const f = makeFx(); + const input = f.edit.addNode('SoundFxInput'); + let tail = input; + for (const effect of effects) { + const node = f.edit.addNode(effect.type, effect.params ?? {}); + f.edit.connect(tail, 'out', node, 'in'); + tail = node; + } + const output = f.edit.addNode('SoundFxOutput'); + f.edit.connect(tail, 'out', output, 'in'); + return { ...f, input, output }; +} + +describe('the sound FX asset', () => { + it('is not placeable, starts with an empty graph and is what the fx picker offers', () => { + expect(isPlaceableType('soundfx')).toBe(false); + expect(buildAsset('soundfx', 'FX')).toMatchObject({ type: 'soundfx', graph: { nodes: [], edges: [] } }); + const f = makeFx(); + f.stack.execute(new CreateSoundCommand('Jump', FILE)); + expect(assetRefChoices(f.project, ['soundfx']).map((a) => a.id)).toEqual([f.fx.id]); + }); + + it('survives a round trip and keeps only sound FX nodes', () => { + const f = line([{ type: 'SoundFxGain', params: { gain: -6 } }]); + expect(roundTrip(f.project)).toEqual(f.project); + + const loaded = deserializeProject({ + name: 'P', + assets: [ + { + id: 'fx', + type: 'soundfx', + name: 'FX', + graph: { + nodes: [ + { id: 'a', type: 'SoundFxInput', x: 0, y: 0, params: {} }, + { id: 'b', type: 'OnStart', x: 0, y: 0, params: {} }, + ], + edges: [], + }, + }, + ], + }); + expect((loaded.assets[0] as SoundFxAsset).graph.nodes.map((n) => n.id)).toEqual(['a']); + }); + + it('carries sound only from a sound port into a sound port', () => { + expect(portsCompatible('audio', 'audio')).toBe(true); + expect(portsCompatible('audio', 'any')).toBe(false); + expect(portsCompatible('number', 'audio')).toBe(false); + expect(portsCompatible('any', 'audio')).toBe(false); + }); + + it('offers its effects, Input and Output once each, and none of them in a behavior', () => { + const f = makeFx(); + const ids = () => paletteEntries(f.project, f.fx).map((e) => e.id); + expect(ids()).toEqual(['SoundFxInput', 'SoundFxOutput', 'SoundFxGain', 'SoundFxPitch', 'SoundFxEq']); + f.edit.addNode('SoundFxInput'); + f.edit.addNode('SoundFxOutput'); + expect(ids()).toEqual(['SoundFxGain', 'SoundFxPitch', 'SoundFxEq']); + + const behavior = makeBehavior(f, 'Logic'); + expect(paletteEntries(f.project, behavior).some((e) => e.def.category === 'sound')).toBe(false); + }); +}); + +describe('compiling a sound FX graph', () => { + it('needs an Output, and an Output fed from the Input', () => { + const f = makeFx(); + expect(compileSoundFx(f.fx.graph)).toMatchObject({ compiled: null, reason: 'No Output node' }); + + const output = f.edit.addNode('SoundFxOutput'); + const gain = f.edit.addNode('SoundFxGain'); + f.edit.connect(gain, 'out', output, 'in'); + const build = compileSoundFx(f.fx.graph); + expect(build.compiled).toBeNull(); + expect(build.problems[gain.id]).toBe('Nothing comes in: connect it to the Input'); + }); + + it('passes the sound on untouched through Input and Output alone', () => { + const build = compileSoundFx(line([]).fx.graph); + expect(build).toMatchObject({ compiled: DRY, effects: 0, reason: '' }); + }); + + it('turns every node on the line into its stages, in order', () => { + const f = line([ + { type: 'SoundFxGain', params: { gain: -6 } }, + { type: 'SoundFxPitch', params: { semitones: 12 } }, + { type: 'SoundFxEq', params: { low: 6, high: -3 } }, + { type: 'SoundFxPitch', params: { semitones: -5 } }, + ]); + const build = compileSoundFx(f.fx.graph); + expect(build.effects).toBe(4); + const chain = build.compiled!; + expect(chain.pitch).toBe(7); + expect(chain.stages).toHaveLength(3); + expect(chain.stages[0]!.kind).toBe('gain'); + expect(chain.stages[0]!.gain).toBeCloseTo(0.501, 3); + expect(chain.stages.slice(1)).toEqual([ + { kind: 'filter', filter: 'lowshelf', frequency: 80, gain: 6, q: 1 }, + { kind: 'filter', filter: 'highshelf', frequency: 12000, gain: -3, q: 1 }, + ]); + }); + + it('leaves out what does nothing and holds a value to its row', () => { + const quiet = compileSoundFx(line([{ type: 'SoundFxGain' }, { type: 'SoundFxEq' }]).fx.graph); + expect(quiet.compiled).toEqual(DRY); + expect(quiet.effects).toBe(2); + + const loud = compileSoundFx(line([{ type: 'SoundFxGain', params: { gain: 200 } }]).fx.graph).compiled!; + expect(loud.stages[0]!.gain).toBeCloseTo(10 ** (24 / 20)); + }); + + it('ignores a node hanging off the line', () => { + const f = line([{ type: 'SoundFxPitch', params: { semitones: 3 } }]); + const aside = f.edit.addNode('SoundFxGain', { gain: -20 }); + f.edit.connect(f.input, 'out', aside, 'in'); + const build = compileSoundFx(f.fx.graph); + expect(build.compiled).toEqual({ pitch: 3, stages: [] }); + expect(build.effects).toBe(1); + }); +}); + +/** Records what it was asked to play, with the line of effects. */ +function fakeAudio(): AudioSink & { played: { file: string; fx: FxChain | undefined }[] } { + const sink = { + played: [] as { file: string; fx: FxChain | undefined }[], + play: (file: string, _volume: number, _loop: boolean, _owner: string, fx?: FxChain) => + sink.played.push({ file, fx }), + stop: () => {}, + stopOwner: () => {}, + }; + return sink; +} + +describe('Play sound with fx', () => { + function playingScene(fxId: (f: ReturnType) => string) { + const scene = makeSpriteScene('Box'); + const withFx = makeFx(scene); + const fx = withFx.fx; + const input = withFx.edit.addNode('SoundFxInput'); + const pitch = withFx.edit.addNode('SoundFxPitch', { semitones: -12 }); + const output = withFx.edit.addNode('SoundFxOutput'); + withFx.edit.connect(input, 'out', pitch, 'in'); + withFx.edit.connect(pitch, 'out', output, 'in'); + const sound = new CreateSoundCommand('Jump', FILE); + scene.stack.execute(sound); + const behavior = makeBehavior(scene, 'Logic'); + place(scene, behavior.id, { x: 0, y: 0 }, place(scene, scene.spriteId, { x: 0, y: 0 })); + const edit = graphEditor(scene, behavior); + const start = edit.addNode('OnStart'); + const play = edit.addNode('PlaySound', { sound: sound.asset.id, fx: fxId(withFx) }); + edit.connect(start, 'exec', play, 'exec'); + return { ...scene, fx, output, pitch }; + } + + it('sends the voice through the asset the node names', () => { + const f = playingScene((s) => s.fx.id); + const audio = fakeAudio(); + startWorld(f, { behaviors: true, audio }).frame(); + expect(audio.played).toEqual([{ file: FILE, fx: { pitch: -12, stages: [] } }]); + }); + + it('plays it dry without an fx, with one that is gone, and with one that does not compile', () => { + const none = fakeAudio(); + startWorld( + playingScene(() => ''), + { behaviors: true, audio: none }, + ).frame(); + const gone = fakeAudio(); + startWorld( + playingScene(() => 'no-such-fx'), + { behaviors: true, audio: gone }, + ).frame(); + const broken = playingScene((s) => s.fx.id); + broken.fx.graph.edges = []; + expect(soundFxBuildFor(broken.fx).compiled).toBeNull(); + const brokenAudio = fakeAudio(); + startWorld(broken, { behaviors: true, audio: brokenAudio }).frame(); + for (const audio of [none, gone, brokenAudio]) expect(audio.played).toEqual([{ file: FILE, fx: DRY }]); + }); +}); diff --git a/tests/helpers.ts b/tests/helpers.ts index 05a62671..89450b39 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -205,7 +205,7 @@ export type FakeContext = NodeContext & { stopped: boolean; changes: { machineId: string; from: string; to: string }[]; /** What the context was asked to play and to silence, in order – nothing here makes a noise. */ - played: { assetId: string; volume: number; loop: boolean | undefined; detached: boolean }[]; + played: { assetId: string; volume: number; loop: boolean | undefined; detached: boolean; fx: string }[]; silenced: string[]; /** The scenes the context was asked to load, in order. */ loaded: string[]; @@ -258,7 +258,8 @@ export function fakeContext(overrides: Partial = {}): FakeContext { played: [], silenced: [], loaded: [], - playSound: (assetId, volume, loop, detached) => ctx.played.push({ assetId, volume, loop, detached }), + playSound: (assetId, volume, loop, detached, fx) => + ctx.played.push({ assetId, volume, loop, detached, fx }), stopSound: (assetId) => ctx.silenced.push(assetId), loadScene: (sceneId) => ctx.loaded.push(sceneId), spawn: () => null, diff --git a/tests/viewport/sound-library.test.ts b/tests/viewport/sound-library.test.ts index 5eb2d98a..c35cf6b9 100644 --- a/tests/viewport/sound-library.test.ts +++ b/tests/viewport/sound-library.test.ts @@ -6,6 +6,7 @@ * The device is stubbed – nothing makes a noise in a unit test, and Web Audio does not exist here. */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { FxChain } from '../../src/core/soundfx/nodes'; import { SoundLibrary } from '../../src/viewport/SoundLibrary'; const FILE = 'a.wav'; @@ -14,6 +15,7 @@ const OTHER = 'b.wav'; interface FakeSource { buffer: unknown; loop: boolean; + playbackRate: { value: number }; onended: (() => void) | null; started: boolean; stopped: boolean; @@ -23,6 +25,29 @@ interface FakeSource { } const sources: FakeSource[] = []; +/** Every `connect` made, as "from > to" by the kind of node – the order a voice is wired in. */ +const wires: string[] = []; +const filters: { + type: string; + frequency: { value: number }; + gain: { value: number }; + Q: { value: number }; +}[] = []; + +/** A node that logs where it is connected to and hands the target back, as Web Audio's does. */ +function wired( + kind: string, + node: T, +): T & { kind: string; connect: (to: unknown) => unknown } { + return { + ...node, + kind, + connect: (to: unknown) => { + wires.push(`${kind} > ${(to as { kind?: string }).kind ?? '?'}`); + return to; + }, + }; +} /** Resolves the pending decodes by hand, so a test decides when the bytes arrive. */ let decodes: (() => void)[] = []; @@ -30,29 +55,39 @@ let decodes: (() => void)[] = []; function fakeContext() { return { state: 'running', - destination: {}, + destination: { kind: 'destination' }, resume: () => Promise.resolve(), suspend: () => Promise.resolve(), decodeAudioData: () => new Promise((resolve) => decodes.push(() => resolve({ duration: 1.5 }))), createBufferSource: () => { - const source: FakeSource = { + const source: FakeSource = wired('source', { buffer: null, loop: false, + playbackRate: { value: 1 }, onended: null, started: false, stopped: false, - connect: (target: unknown) => target, start: () => { source.started = true; }, stop: () => { source.stopped = true; }, - }; + }); sources.push(source); return source; }, - createGain: () => ({ gain: { value: 1 }, connect: (target: unknown) => target }), + createGain: () => wired('gain', { gain: { value: 1 } }), + createBiquadFilter: () => { + const filter = wired('filter', { + type: '', + frequency: { value: 0 }, + gain: { value: 0 }, + Q: { value: 0 }, + }); + filters.push(filter); + return filter; + }, }; } @@ -61,6 +96,8 @@ const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); beforeEach(() => { sources.length = 0; + wires.length = 0; + filters.length = 0; decodes = []; vi.stubGlobal( 'AudioContext', @@ -80,8 +117,8 @@ afterEach(() => { }); /** Plays `file` and lets its bytes arrive, unless `hold` keeps the decode pending. */ -async function play(library: SoundLibrary, file = FILE, { hold = false } = {}): Promise { - library.play(file, 1, false); +async function play(library: SoundLibrary, file = FILE, { hold = false } = {}, fx?: FxChain): Promise { + library.play(file, 1, false, '', fx); await settle(); if (hold) return; for (const resolve of decodes.splice(0)) resolve(); @@ -104,7 +141,31 @@ describe('the sound library', () => { await settle(); for (const resolve of decodes.splice(0)) resolve(); await settle(); - expect(library.stats().last).toEqual({ file: FILE, volume: 0.25, loop: true }); + expect(library.stats().last).toEqual({ file: FILE, volume: 0.25, loop: true, pitch: 0, stages: 0 }); + }); + + it('plays a dry voice straight into its volume', async () => { + await play(new SoundLibrary()); + expect(wires).toEqual(['source > gain', 'gain > destination']); + expect(sources[0]?.playbackRate.value).toBe(1); + }); + + it('sends a voice through its sound FX in order, pitched at the source (#311)', async () => { + const library = new SoundLibrary(); + const fx: FxChain = { + pitch: 12, + stages: [ + { kind: 'gain', gain: 0.5 }, + { kind: 'filter', filter: 'lowshelf', frequency: 80, gain: 6, q: 1 }, + ], + }; + await play(library, FILE, {}, fx); + expect(wires).toEqual(['source > gain', 'gain > filter', 'filter > gain', 'gain > destination']); + expect(sources[0]?.playbackRate.value).toBe(2); + expect(filters).toMatchObject([ + { type: 'lowshelf', frequency: { value: 80 }, gain: { value: 6 }, Q: { value: 1 } }, + ]); + expect(library.stats().last).toMatchObject({ pitch: 12, stages: 2 }); }); it('a stop while the bytes are in flight keeps the sound from starting afterwards', async () => { diff --git a/tools/e2e/34-sound.mjs b/tools/e2e/34-sound.mjs index 2b1f338b..3e226280 100644 --- a/tools/e2e/34-sound.mjs +++ b/tools/e2e/34-sound.mjs @@ -270,6 +270,92 @@ export async function sound(t) { ); await setDetached(false); + // 34g-3. A sound FX asset (#311): "+ → Sound FX" opens its tab, which says what keeps the graph + // from compiling; built into Input → Pitch → Gain → Output, the header plays the sound + // through it, and a `Play sound` naming it sends the game's voice the same way. + await showTab('scene'); + await clickText('+'); + await pickAssetType(t, 'Sound FX'); + const fxCreated = await json(`{ + tab: __gm.tab(), + current: __gm.soundfx.current(), + info: __gm.soundfx.info(), + preview: __gm.soundfx.preview(), + assets: __gm.state.project.assets.filter((a) => a.type === 'soundfx').map((a) => a.id), + }`); + check( + 'the + menu creates a sound FX asset and opens its tab, with no Output to compile and no Play', + fxCreated.tab === 'soundfx' && + fxCreated.assets.length === 1 && + fxCreated.current === fxCreated.assets[0] && + fxCreated.info === 'No Output node' && + fxCreated.preview.sound === soundId && + fxCreated.preview.play === false, + JSON.stringify(fxCreated), + ); + const fxId = fxCreated.assets[0]; + const fxCanvas = await json('__gm.soundfx.canvasRect()'); + await click(fxCanvas.x + 80, fxCanvas.y + 60, 'right'); + const fxPalette = await json('__gm.overlayTexts()'); + check( + 'its palette offers the sound effects and no behavior node', + ['Input', 'Output', 'Gain', 'Pitch', 'EQ'].every((label) => fxPalette.includes(label)) && + !fxPalette.includes('Branch'), + JSON.stringify(fxPalette.slice(0, 12)), + ); + await press('Escape'); + await mutate(` + const fx = __gm.state.project.assets.find((a) => a.id === ${JSON.stringify(fxId)}); + fx.name = 'Low'; + fx.graph = ${JSON.stringify({ + nodes: [ + node('fx-in', 'SoundFxInput', 0, 0), + node('fx-pitch', 'SoundFxPitch', 220, 0, { semitones: -12 }), + node('fx-gain', 'SoundFxGain', 440, 0, { gain: -6 }), + node('fx-out', 'SoundFxOutput', 660, 0), + ], + edges: [ + edge('fx-e1', 'fx-in', 'out', 'fx-pitch', 'in'), + edge('fx-e2', 'fx-pitch', 'out', 'fx-gain', 'in'), + edge('fx-e3', 'fx-gain', 'out', 'fx-out', 'in'), + ], + })};`); + const fxBuilt = await json('{ info: __gm.soundfx.info(), preview: __gm.soundfx.preview() }'); + check( + 'Input → Pitch → Gain → Output compiles to two effects, and Play is offered', + fxBuilt.info === '2 effects' && fxBuilt.preview.play === true, + JSON.stringify(fxBuilt), + ); + await evaluate('__gm.sound.stopAll()'); + const beforeFx = await stats(); + const fxPlayAt = await json('__gm.soundfx.playAt()'); + await click(fxPlayAt.x, fxPlayAt.y); + const heardFx = await waitFor(stats, (s) => s.started > beforeFx.started); + check( + 'the tab’s Play sends the sound through the graph: an octave down and one gain stage', + heardFx.last?.file === blip.file && heardFx.last.pitch === -12 && heardFx.last.stages === 1, + JSON.stringify(heardFx), + ); + await evaluate('__gm.sound.stopAll()'); + await showTab('scene'); + await mutate(` + const graph = __gm.state.project.assets.find((a) => a.id === 'sn-logic').graph; + Object.assign(graph.nodes.find((n) => n.id === 'sn-play').params, { loop: 'false', fx: ${JSON.stringify(fxId)} });`); + const beforeGameFx = await stats(); + await play(); + const gameFx = await waitFor(stats, (s) => s.started > beforeGameFx.started); + check( + 'a Play sound naming the asset plays the game’s voice through it', + gameFx.last?.pitch === -12 && gameFx.last.stages === 1, + JSON.stringify(gameFx), + ); + await stop(); + await mutate(` + const graph = __gm.state.project.assets.find((a) => a.id === 'sn-logic').graph; + Object.assign(graph.nodes.find((n) => n.id === 'sn-play').params, { loop: '', fx: '' }); + const project = __gm.state.project; + project.assets = project.assets.filter((a) => a.id !== ${JSON.stringify(fxId)});`); + // 34h. Dropping a sound file onto the editor uploads it and makes an asset named after the file. const dropped = await evaluate(` (async () => {