diff --git a/src/core/commands/graphCommands.ts b/src/core/commands/graphCommands.ts index f7eebfae..9aaddb9d 100644 --- a/src/core/commands/graphCommands.ts +++ b/src/core/commands/graphCommands.ts @@ -1,16 +1,8 @@ -import { - edgesInto, - edgesOfNode, - findNode, - type Graph, - type GraphEdge, - type GraphNode, - portsCompatible, -} from '../graph/Graph'; +import { edgesInto, edgesOfNode, findNode, type Graph, type GraphEdge, type GraphNode } from '../graph/Graph'; import type { GraphLookup } from '../graph/lookup'; import { isTerminalType, resolveDef, terminalNode } from '../graph/nested'; import { nextTransitionName, parallelTransitions } from '../graph/stateMachine'; -import { nodeDef, takesManyEdges } from '../graph/NodeRegistry'; +import { nodeDef, portsMeet, takesManyEdges } from '../graph/NodeRegistry'; import { graphLookup } from '../model/graphLookup'; import { newId } from '../ids'; import type { PropertyBag, PropertyValue } from '../model/Properties'; @@ -221,8 +213,7 @@ export class ConnectCommand implements Command { const toDef = resolveDef(toNode, graph, lookup); const inp = toDef?.inputs.find((p) => p.key === this.edge.to.port); if (!out || !inp) throw new CommandError('Port not found'); - if (!portsCompatible(out.type, inp.type)) - throw new CommandError(`Cannot connect ${out.type} to ${inp.type}`); + if (!portsMeet(out, inp)) throw new CommandError(`Cannot connect ${out.type} to ${inp.type}`); if (graph.edges.some((e) => sameEnds(e, this.edge))) { // Transitions are told apart by name, so a further one between the same states gets its own. if (fromDef?.transitionEdges !== true || toDef?.transitionEdges !== true) diff --git a/src/core/graph/Graph.ts b/src/core/graph/Graph.ts index df1926eb..fb312579 100644 --- a/src/core/graph/Graph.ts +++ b/src/core/graph/Graph.ts @@ -9,6 +9,7 @@ export type { DeclaredPortKind } from '../properties/kinds'; * `picture` is what a shader's Pass draws (#301), for the nodes that read a picture. `fxparams` is * the values a `Play sound` hands to the parameters of its sound FX (#311). `trigger` is a note * arriving at a source of an instrument (#331): on for the notes it stands for, off for all others. + * `envelope` is the curve of an Envelope (#335), moving a number of a note while it plays. */ export type PortType = | 'exec' @@ -23,6 +24,7 @@ export type PortType = | 'audio' | 'fxparams' | 'trigger' + | 'envelope' | 'any'; export interface PortRef { @@ -190,7 +192,8 @@ export function graphFingerprint(graph: Graph): string { * A number may feed a colour: the shader compiler spreads it over the three channels. A picture is * no value a calculation could take, so like `exec` it goes only where a picture is read – and a * sound on its way through effects (#311) only where a sound goes in, and its parameters only to a - * `Play sound`. + * `Play sound`. An envelope goes into a number only where that number says it can follow one + * (`portsMeet`). */ export function portsCompatible(from: PortType, to: PortType): boolean { if (from === 'exec' || to === 'exec') return from === to; @@ -198,6 +201,7 @@ export function portsCompatible(from: PortType, to: PortType): boolean { if (from === 'audio' || to === 'audio') return from === to; if (from === 'fxparams' || to === 'fxparams') return from === to; if (from === 'trigger' || to === 'trigger') return from === to; + if (from === 'envelope' || to === 'envelope') return from === to; if (from === 'number' && to === 'vec3') return true; return from === to || from === 'any' || to === 'any'; } diff --git a/src/core/graph/NodeRegistry.ts b/src/core/graph/NodeRegistry.ts index 56d45c2a..2d00db8f 100644 --- a/src/core/graph/NodeRegistry.ts +++ b/src/core/graph/NodeRegistry.ts @@ -1,7 +1,7 @@ import { isList, type ListValue, type PropertyBag, type PropertyValue } from '../model/Properties'; import { type DeclaredPortKind, isKindWith, plainKindOf } from '../properties/kinds'; import { flatDef, type PropertyDef } from '../properties/PropertySchema'; -import type { DeclaredPort, Graph, GraphNode, PortType } from './Graph'; +import { type DeclaredPort, type Graph, type GraphNode, type PortType, portsCompatible } from './Graph'; import type { GraphLookup, ValueTyping } from './lookup'; import type { NodeContext } from './NodeContext'; import type { SharedPortGroup } from './portTypes'; @@ -29,6 +29,17 @@ export interface PortDef { typing?: ValueTyping; /** An input taking any number of edges, like an exec input: the layers of an instrument (#331). */ many?: true; + /** A number input an Envelope's curve can move while a note plays (#335). */ + followsEnvelope?: true; +} + +/** Whether an output may feed an input: by their types, and an envelope only where the number follows one. */ +export function portsMeet( + out: Pick, + inp: Pick, +): boolean { + if (out.type === 'envelope' && inp.type === 'number') return inp.followsEnvelope === true; + return portsCompatible(out.type, inp.type); } /** @@ -227,7 +238,8 @@ function isInlinePort(port: PortDef): boolean { port.type !== 'picture' && port.type !== 'audio' && port.type !== 'fxparams' && - port.type !== 'trigger' + port.type !== 'trigger' && + port.type !== 'envelope' ); } diff --git a/src/core/graph/dropTarget.ts b/src/core/graph/dropTarget.ts index 6b1ac934..e985d6d6 100644 --- a/src/core/graph/dropTarget.ts +++ b/src/core/graph/dropTarget.ts @@ -1,7 +1,7 @@ -import { edgesInto, findNode, type Graph, type PortType, portsCompatible } from './Graph'; +import { edgesInto, findNode, type Graph, type PortType } from './Graph'; import type { GraphLookup } from './lookup'; import { resolveDef } from './nested'; -import { type NodeDef, takesManyEdges } from './NodeRegistry'; +import { type NodeDef, type PortDef, portsMeet, takesManyEdges } from './NodeRegistry'; /** One end of an edge being dragged: a port on a node, seen from its own side. */ export interface PortSide { @@ -10,21 +10,30 @@ export interface PortSide { isOutput: boolean; } -export function portTypeOf(graph: Graph, side: PortSide, lookup: GraphLookup): PortType | undefined { +/** What a drag starts from: a port, or just its type where nothing else about it counts. */ +type FromPort = PortType | Pick; + +const asPort = (from: FromPort) => (typeof from === 'string' ? { type: from } : from); + +export function portOf(graph: Graph, side: PortSide, lookup: GraphLookup): PortDef | undefined { const node = findNode(graph, side.nodeId); const def = node && resolveDef(node, graph, lookup); const list = side.isOutput ? def?.outputs : def?.inputs; - return list?.find((p) => p.key === side.port)?.type; + return list?.find((p) => p.key === side.port); +} + +export function portTypeOf(graph: Graph, side: PortSide, lookup: GraphLookup): PortType | undefined { + return portOf(graph, side, lookup)?.type; } /** Whether a drag from `from` may land on `to`, i.e. whether ConnectCommand would accept it. */ export function canConnect(graph: Graph, from: PortSide, to: PortSide, lookup: GraphLookup): boolean { if (from.isOutput === to.isOutput || from.nodeId === to.nodeId) return false; - const fromType = portTypeOf(graph, from, lookup); - const toType = portTypeOf(graph, to, lookup); - if (fromType === undefined || toType === undefined) return false; - const [out, inp] = from.isOutput ? [fromType, toType] : [toType, fromType]; - return portsCompatible(out, inp) && !linked(graph, from, to); + const fromPort = portOf(graph, from, lookup); + const toPort = portOf(graph, to, lookup); + if (fromPort === undefined || toPort === undefined) return false; + const [out, inp] = from.isOutput ? [fromPort, toPort] : [toPort, fromPort]; + return portsMeet(out, inp) && !linked(graph, from, to); } /** @@ -41,9 +50,9 @@ export function dropTargetPort( if (nodeId === from.nodeId) return null; const node = findNode(graph, nodeId); const def = node && resolveDef(node, graph, lookup); - const fromType = portTypeOf(graph, from, lookup); - if (!def || fromType === undefined) return null; - const candidates = compatiblePorts(def, fromType, from.isOutput) + const fromPort = portOf(graph, from, lookup); + if (!def || fromPort === undefined) return null; + const candidates = compatiblePorts(def, fromPort, from.isOutput) .map((p) => ({ side: { nodeId, port: p.key, isOutput: !from.isOutput }, many: takesManyEdges(p) })) .filter((c) => !linked(graph, from, c.side)); const free = candidates.find((c) => isFree(graph, c.side, c.many)); @@ -54,23 +63,22 @@ export function dropTargetPort( * Whether a node of this type has any port a drag from `fromType` could land on – the filter for * the palette that opens when an edge is dropped into empty space. */ -export function defAcceptsFrom(def: NodeDef, fromType: PortType, fromIsOutput: boolean): boolean { - return compatiblePorts(def, fromType, fromIsOutput).length > 0; +export function defAcceptsFrom(def: NodeDef, from: FromPort, fromIsOutput: boolean): boolean { + return compatiblePorts(def, from, fromIsOutput).length > 0; } /** * The port a *fresh* node of this type offers such a drag. Every port of a node that is not in the * graph yet is free, so the first compatible one is what `dropTargetPort` would pick for it. */ -export function firstPortFor(def: NodeDef, fromType: PortType, fromIsOutput: boolean): string | null { - return compatiblePorts(def, fromType, fromIsOutput)[0]?.key ?? null; +export function firstPortFor(def: NodeDef, from: FromPort, fromIsOutput: boolean): string | null { + return compatiblePorts(def, from, fromIsOutput)[0]?.key ?? null; } -function compatiblePorts(def: NodeDef, fromType: PortType, fromIsOutput: boolean) { +function compatiblePorts(def: NodeDef, from: FromPort, fromIsOutput: boolean) { + const port = asPort(from); const ports = fromIsOutput ? def.inputs : def.outputs; - return ports.filter((p) => - fromIsOutput ? portsCompatible(fromType, p.type) : portsCompatible(p.type, fromType), - ); + return ports.filter((p) => (fromIsOutput ? portsMeet(port, p) : portsMeet(p, port))); } /** Only a data input is exclusive – exec and `many` inputs and every output take any number of edges. */ diff --git a/src/core/graph/nodes/action/StopSound.ts b/src/core/graph/nodes/action/StopSound.ts index 9a51b0c0..3fab4127 100644 --- a/src/core/graph/nodes/action/StopSound.ts +++ b/src/core/graph/nodes/action/StopSound.ts @@ -11,7 +11,8 @@ export const StopSound: NodeDef = { 'With no sound picked it silences everything the run is playing. With "voice" connected it ' + 'silences only that one play, the number Play sound handed out, and nothing where it is 0. ' + 'A voice fades out in a few milliseconds, echoes and all; with "let ring" only the sound stops ' + - 'and its echoes fade as they would.', + 'and its echoes fade as they would, and a note of an instrument with an Envelope falls away in ' + + 'its release first.', inputs: [ exec(), soundPort(), diff --git a/src/core/instrument/compile.ts b/src/core/instrument/compile.ts index 07af1897..0f9195cd 100644 --- a/src/core/instrument/compile.ts +++ b/src/core/instrument/compile.ts @@ -1,5 +1,12 @@ import { graphEmitter } from '../graph/emitGraph'; -import { type DeclaredPort, edgesInto, type Graph, graphFingerprint, type GraphNode } from '../graph/Graph'; +import { + type DeclaredPort, + edgesInto, + edgesOutOf, + type Graph, + graphFingerprint, + type GraphNode, +} from '../graph/Graph'; import type { GraphLookup } from '../graph/lookup'; import { resolveDef, terminalNode } from '../graph/nested'; import { inlineValue, type NodeDef } from '../graph/NodeRegistry'; @@ -13,8 +20,11 @@ import { volumeOf } from '../properties/sounds'; import { soundFxBuildFor } from '../soundfx/compile'; import { type FxAmount, + type FxEnvelopePlan, + type FxModPlan, type FxPlan, type FxStagePlan, + held, isSoundFxNodeDef, SOUND_FX_INPUT, SOUND_FX_OUTPUT, @@ -24,6 +34,8 @@ import { withStages, } from '../soundfx/nodes'; import { + ENVELOPE_ROWS, + INSTRUMENT_ENVELOPE, INSTRUMENT_NOTES, INSTRUMENT_OUTPUT, INSTRUMENT_PARAMETERS, @@ -84,9 +96,9 @@ export interface InstrumentBuild { /** * What travels along an edge of an instrument: sounds (none where nothing plays for this note), a - * number, or whether a trigger is on for this note. + * number, whether a trigger is on for this note, or the curve of an Envelope (#335). */ -type Value = { plans: FxPlan[] } | { amount: FxAmount } | { on: boolean }; +type Value = { plans: FxPlan[] } | { amount: FxAmount } | { on: boolean } | { curve: FxModPlan }; const SILENT: Value = { plans: [] }; @@ -117,6 +129,8 @@ export function compileInstrument(graph: Graph, note: number, sources: Instrumen emit: (def, inputs, node) => emitNode(def, inputs, node), unfed: (node, port) => { if (port.type === 'number') return { amount: { value: asNumber(inlineValue(node, port)) } }; + // An Envelope that only moves numbers has no sound to shape. + if (node.type === INSTRUMENT_ENVELOPE && edgesOutOf(graph, node.id, 'out').length === 0) return SILENT; walk.problem( node, port.type === 'trigger' @@ -126,6 +140,12 @@ export function compileInstrument(graph: Graph, note: number, sources: Instrumen return null; }, conform: (node, port, value) => { + if ('curve' in value) { + if (port.type === 'number' && port.followsEnvelope) + return { amount: { value: value.curve.from.value, mod: value.curve } }; + walk.problem(node, `"${port.label}" cannot follow an Envelope`); + return null; + } const fits = port.type === 'audio' ? 'plans' in value @@ -170,6 +190,18 @@ export function compileInstrument(graph: Graph, note: number, sources: Instrumen const arriving = plansAt('in'); return { out: { plans: fx ? arriving.map((plan) => throughFx(fx, plan, amounts)) : arriving } }; } + case INSTRUMENT_ENVELOPE: { + const envelope = envelopeOf(node, amounts); + const curve = { + envelope, + from: held(amounts.from, ENVELOPE_ROWS.from), + to: held(amounts.to, ENVELOPE_ROWS.to), + }; + return { + out: { plans: plansAt('in').map((plan) => withStages(plan, [{ kind: 'envelope', envelope }])) }, + value: { curve }, + }; + } case INSTRUMENT_OUTPUT: return { out: { plans: plansAt('in') } }; } @@ -222,6 +254,18 @@ export function compileInstrument(graph: Graph, note: number, sources: Instrumen return { layers: arrived.plans, problems: walk.problems, reason: '' }; } +/** An Envelope's times and level, held to its rows; one fed by a parameter is filled in as its note starts. */ +function envelopeOf(node: GraphNode, amounts: Readonly>): FxEnvelopePlan { + const rows = ENVELOPE_ROWS; + return { + id: node.id, + attack: held(amounts.attack, rows.attack), + decay: held(amounts.decay, rows.decay), + sustain: held(amounts.sustain, rows.sustain), + release: held(amounts.release, rows.release), + }; +} + /** The outputs of a Notes node for one note: its triggers, and the note as numbers. */ function notesOutputs(node: GraphNode, note: number): Record { const { from, to } = notesRange(node.params); @@ -268,6 +312,9 @@ function throughFx(fx: FxPlan, sound: FxPlan, given: Readonly FxAmount): FxStagePlan { if (stage.kind === 'gain') return { ...stage, db: bind(stage.db) }; if (stage.kind === 'filter') return { ...stage, gain: bind(stage.gain) }; + if (stage.kind === 'pass') + return { ...stage, cutoff: bind(stage.cutoff), resonance: bind(stage.resonance) }; + if (stage.kind === 'envelope') return stage; return { ...stage, time: bind(stage.time), diff --git a/src/core/instrument/envelope.ts b/src/core/instrument/envelope.ts new file mode 100644 index 00000000..7e594803 --- /dev/null +++ b/src/core/instrument/envelope.ts @@ -0,0 +1,51 @@ +import type { FxEnvelope, FxMod } from '../soundfx/nodes'; + +/** + * How many time constants a decay or a release takes for its time: ln 1000, so it is 60 dB of the + * way there when its time is up – where a tail counts as gone elsewhere too (#324). + */ +export const SETTLE = Math.log(1000); + +/** Seconds a release takes at least, so a note ending on a release of 0 does not click. */ +export const SHORTEST_RELEASE = 0.01; + +export function releaseTime(envelope: FxEnvelope): number { + return Math.max(SHORTEST_RELEASE, envelope.release); +} + +/** + * The level of an Envelope `t` seconds after its note started, while the note is held: a straight + * line up to 1 in `attack`, then down towards `sustain` the way a decay falls, by `SETTLE` time + * constants in `decay`. What a release starts from when the note ends early. + */ +export function envelopeLevel(envelope: FxEnvelope, t: number): number { + if (t <= 0) return 0; + if (t < envelope.attack) return t / envelope.attack; + if (envelope.decay <= 0) return envelope.sustain; + const into = t - envelope.attack; + return envelope.sustain + (1 - envelope.sustain) * Math.exp((-into * SETTLE) / envelope.decay); +} + +/** Points of a `modCurve` across the input -1…1: about 500 steps for the half a curve reads. */ +const CURVE_POINTS = 1025; + +/** + * What a WaveShaper makes of an Envelope's level (0 to 1) to move a number: `from` at 0, `to` at 1, + * evenly between – or in equal ratios where `ratio` (a cutoff, a volume), as long as both ends are + * above 0. `map` turns the number into what the node holds. A WaveShaper reads -1…1; below 0 an + * Envelope never goes, so that half stays at `from`. + */ +export function modCurve( + mod: FxMod, + ratio: boolean, + map = (value: number) => value, +): Float32Array { + const { from, to } = mod; + const inRatios = ratio && from > 0 && to > 0; + const curve = new Float32Array(CURVE_POINTS); + for (let i = 0; i < CURVE_POINTS; i++) { + const s = Math.max(0, (2 * i) / (CURVE_POINTS - 1) - 1); + curve[i] = map(inRatios ? from * (to / from) ** s : from + (to - from) * s); + } + return curve; +} diff --git a/src/core/instrument/nodes.ts b/src/core/instrument/nodes.ts index 6cbf1cd1..4fd95882 100644 --- a/src/core/instrument/nodes.ts +++ b/src/core/instrument/nodes.ts @@ -9,7 +9,7 @@ import { } from '../graph/nested'; import type { NodeDef, PortDef } from '../graph/NodeRegistry'; import { type PropertyBag, textOf } from '../model/Properties'; -import type { PropertyDef } from '../properties/PropertySchema'; +import type { FlatDef, PropertyDef } from '../properties/PropertySchema'; import { soundFxParamPort } from '../soundfx/nodes'; import { HIGHEST_NOTE, heldNote, LOWEST_NOTE, MIDDLE_C, NOTE_OPTIONS, noteName } from './notes'; @@ -228,6 +228,40 @@ export const InstrumentSoundFx: InstrumentNodeDef = { }, }; +export const INSTRUMENT_ENVELOPE = 'InstrumentEnvelope'; + +function seconds(key: string, label: string, value: number): FlatDef<'number'> { + return { key, label, kind: 'number', default: value, min: 0, max: 30, step: 0.001 }; +} + +export const ENVELOPE_ROWS = { + attack: seconds('attack', 'Attack (s)', 0.005), + decay: seconds('decay', 'Decay (s)', 0.1), + sustain: { key: 'sustain', label: 'Sustain', kind: 'number', default: 1, min: 0, max: 1, step: 0.01 }, + release: seconds('release', 'Release (s)', 0.1), + from: { key: 'from', label: 'From', kind: 'number', default: 0, step: 0.01 }, + to: { key: 'to', label: 'To', kind: 'number', default: 1, step: 0.01 }, +} satisfies Record>; + +const numberIn = (row: FlatDef<'number'>): PortDef => ({ + key: row.key, + label: row.label, + type: 'number', + field: row, +}); + +export const InstrumentEnvelope: InstrumentNodeDef = { + type: INSTRUMENT_ENVELOPE, + label: 'Envelope', + category: 'sound', + instrument: true, + description: + 'Shapes a note as it is played: from silence up to full in "attack" seconds, down to the level "sustain" (0 to 1) in "decay", held there while the note is, and back to silence in "release" once it ends. The rise is a straight line, the falls ease out the way a struck string does. On the way of the sound it is the volume: a Gate or a Loop plays on through the release, and the note lasts until it has faded. A One-shot under it alone plays through as it would. "value" moves a number with the same curve, from "from" at rest to "to" at the top: the cutoff or resonance of a Filter, a Pitch, a Gain, the share of a Mix, a band of the EQ. For a curve of its own, take a second Envelope and leave its "in" open. The numbers count from the moment the note starts.', + inputs: [audioIn, ...Object.values(ENVELOPE_ROWS).map(numberIn)], + outputs: [audioOut, { key: 'value', label: 'value', type: 'envelope' }], + params: [], +}; + export const InstrumentOutput: InstrumentNodeDef = { type: INSTRUMENT_OUTPUT, label: 'Output', @@ -245,6 +279,7 @@ export const INSTRUMENT_NODES: readonly InstrumentNodeDef[] = [ InstrumentSample, InstrumentParameters, InstrumentSoundFx, + InstrumentEnvelope, InstrumentOutput, ]; diff --git a/src/core/soundfx/nodes.ts b/src/core/soundfx/nodes.ts index 303d6d96..a75a1909 100644 --- a/src/core/soundfx/nodes.ts +++ b/src/core/soundfx/nodes.ts @@ -10,15 +10,64 @@ export { SOUND_FX_INPUT, SOUND_FX_OUTPUT }; /** The filter shapes the EQ uses – Web Audio's names, so the player hands them on as they are. */ export type FxFilter = 'lowshelf' | 'peaking' | 'highshelf'; +/** The shapes of a Filter (#335). */ +export type FxPassFilter = 'lowpass' | 'highpass' | 'bandpass'; + /** - * One step a voice passes on its way out: a volume, one band of an equalizer, or an echo (#327). An - * echo's `damping` is already the cutoff of the low-pass in its feedback, in Hz. + * One step a voice passes on its way out: a volume, one band of an equalizer, a filter, the volume an + * Envelope gives a note (#335), or an echo (#327). An echo's `damping` is already the cutoff of the + * low-pass in its feedback, in Hz. A `mod` is an Envelope moving that number while the note plays. */ export type FxStage = - | { kind: 'gain'; gain: number } - | { kind: 'filter'; filter: FxFilter; frequency: number; gain: number; q: number } + | { kind: 'gain'; gain: number; mod?: FxMod } + | { kind: 'filter'; filter: FxFilter; frequency: number; gain: number; q: number; mod?: FxMod } + | FxPass + | { kind: 'envelope'; envelope: FxEnvelope } | FxDelay; +/** A filter as Web Audio takes it: `q` in dB for a low- or high-pass, as a factor for a band-pass. */ +export interface FxPass { + kind: 'pass'; + filter: FxPassFilter; + cutoff: number; + q: number; + cutoffMod?: FxMod; + qMod?: FxMod; +} + +/** + * An Envelope as a note starts it (#335): seconds up to the top, down to `sustain` (0 to 1) and back + * to silence once the note ends. `id` is its node, so every number it moves follows one curve. + */ +export interface FxEnvelope { + id: string; + attack: number; + decay: number; + sustain: number; + release: number; +} + +/** A number an Envelope moves: `from` at rest, before the note and after its release, `to` at the top. */ +export interface FxMod { + envelope: FxEnvelope; + from: number; + to: number; +} + +export interface FxEnvelopePlan { + id: string; + attack: FxAmount; + decay: FxAmount; + sustain: FxAmount; + release: FxAmount; +} + +export interface FxModPlan { + envelope: FxEnvelopePlan; + from: FxAmount; + to: FxAmount; +} + export interface FxDelay { kind: 'delay'; time: number; @@ -49,11 +98,18 @@ export interface FxSource { * says the way has a Pitch at all – at 0 too, when that comes from a parameter – and so is played * by the pitch worklet (#317). Behind a Mix (#316) it is two sounds, `b` making up the share `mix` * of what goes on through `stages` – or with the curve `both` each one full up to the middle and - * only fading on the other half. + * only fading on the other half. `pitchMods` and `mixMod` are Envelopes moving them (#335), on top + * of `pitch` and in place of `mix`. */ export type FxChain = - | { pitch: number; shifted: boolean; stages: readonly FxStage[]; source?: FxSource } - | { mix: number; curve?: MixCurve; a: FxChain; b: FxChain; stages: readonly FxStage[] }; + | { + pitch: number; + shifted: boolean; + stages: readonly FxStage[]; + source?: FxSource; + pitchMods?: readonly FxMod[]; + } + | { mix: number; curve?: MixCurve; a: FxChain; b: FxChain; stages: readonly FxStage[]; mixMod?: FxMod }; /** How a Mix shares its two sounds out: `crossfade` (left out of a plan) or `both` at the middle. */ export type MixCurve = 'crossfade' | 'both'; @@ -63,18 +119,22 @@ export const DRY: FxChain = { pitch: 0, shifted: false, stages: [] }; /** * A number of the graph as the compiler knows it: typed in, or a parameter of the Input that a * `Play sound` may hand a value for (#311). `value` is what it is otherwise, and a parameter is held - * to the row it ends in only once its value is known. + * to the row it ends in only once its value is known. With `mod` an Envelope moves it (#335), and + * `value` is where it rests. */ export interface FxAmount { value: number; param?: string; min?: number; max?: number; + mod?: FxModPlan; } export type FxStagePlan = | { kind: 'gain'; db: FxAmount } | { kind: 'filter'; filter: FxFilter; frequency: number; gain: FxAmount; q: number } + | { kind: 'pass'; filter: FxPassFilter; cutoff: FxAmount; resonance: FxAmount } + | { kind: 'envelope'; envelope: FxEnvelopePlan } | { kind: 'delay'; /** Seconds, or quarter notes where `beats` (#332). */ @@ -138,21 +198,37 @@ function numberIn(row: FlatDef<'number'>): PortDef { return { key: row.key, label: row.label, type: 'number', field: row }; } +/** A number the audio thread holds, which an Envelope can move while a note plays (#335). */ +function followingIn(row: FlatDef<'number'>): PortDef { + return { ...numberIn(row), followsEnvelope: true }; +} + function holdTo(value: number, row: FlatDef<'number'>): number { if (!Number.isFinite(value)) return row.default; return Math.max(row.min ?? -Infinity, Math.min(row.max ?? Infinity, value)); } -/** The amount at an input, held to its row, so a hand-edited project cannot blow a speaker. */ -function held(amount: FxAmount | undefined, row: FlatDef<'number'>): FxAmount { +/** + * The amount at an input, held to its row, so a hand-edited project cannot blow a speaker. An + * Envelope's two ends are held there as well. + */ +export function held(amount: FxAmount | undefined, row: FlatDef<'number'>): FxAmount { if (!amount) return { value: row.default }; + if (amount.mod) { + const from = held(amount.mod.from, row); + return { value: from.value, mod: { ...amount.mod, from, to: held(amount.mod.to, row) } }; + } const value = holdTo(amount.value, row); if (amount.param === undefined) return { value }; return { value, param: amount.param, min: row.min, max: row.max }; } /** Whether an amount does nothing however the sound is played: a typed-in 0. */ -const isNone = (amount: FxAmount) => amount.param === undefined && amount.value === 0; +const isNone = (amount: FxAmount) => + amount.param === undefined && amount.mod === undefined && amount.value === 0; + +/** Whether an amount is a number typed in, so a node can tell what it does already. */ +const isFixed = (amount: FxAmount) => amount.param === undefined && amount.mod === undefined; const arriving = (sounds: Readonly>) => sounds.in ?? EMPTY_PLAN; @@ -205,7 +281,7 @@ export const SoundFxGain: SoundFxNodeDef = { category: 'sound', description: 'Makes the sound louder or quieter, in decibels: 0 leaves it as it is, -6 is about half as loud, +6 twice. Above 0 it can clip.', - inputs: [audioIn, numberIn(GAIN)], + inputs: [audioIn, followingIn(GAIN)], outputs: [audioOut], params: [], emit: (sounds, amounts) => { @@ -222,7 +298,7 @@ export const SoundFxPitch: SoundFxNodeDef = { category: 'sound', description: 'Plays the sound higher or lower, in semitones: 12 is an octave up, 0.5 a quarter tone. The sound keeps its length, also when the pitch changes while it plays. The pitch is made where the sound is read, so an effect before the Pitch acts as if it came after it.', - inputs: [audioIn, numberIn(SEMITONES)], + inputs: [audioIn, followingIn(SEMITONES)], outputs: [audioOut], params: [], emit: (sounds, amounts) => { @@ -250,7 +326,7 @@ export const SoundFxEq: SoundFxNodeDef = { category: 'sound', description: 'A five-band equalizer: raises or lowers the lows (80 Hz), low mids (250 Hz), mids (1 kHz), high mids (4 kHz) and highs (12 kHz), each in decibels. A band at 0 does nothing.', - inputs: [audioIn, ...EQ_ROWS.map(numberIn)], + inputs: [audioIn, ...EQ_ROWS.map(followingIn)], outputs: [audioOut], params: [], emit: (sounds, amounts) => { @@ -264,6 +340,60 @@ export const SoundFxEq: SoundFxNodeDef = { }, }; +/** The Q of a low- or high-pass with no bump at its cutoff, in the decibels Web Audio takes it in. */ +export const FLAT_Q = 20 * Math.log10(Math.SQRT1_2); +/** How far a low- or high-pass at resonance 1 rises at its cutoff, in dB: it rings, but stays usable. */ +const RESONANT_Q = 20; +/** A band-pass from about two octaves wide at resonance 0 to a narrow ring at 1, as a factor. */ +const WIDE_BAND_Q = 0.7; +const NARROW_BAND_Q = 20; + +/** The Q a Filter of that shape has at a resonance from 0 to 1. */ +export function passQ(filter: FxPassFilter, resonance: number): number { + const r = Math.max(0, Math.min(1, resonance)); + if (filter === 'bandpass') return WIDE_BAND_Q * (NARROW_BAND_Q / WIDE_BAND_Q) ** r; + return FLAT_Q + (RESONANT_Q - FLAT_Q) * r; +} + +const CUTOFF = { ...numberRow('cutoff', 'Cutoff (Hz)', 20, 20000, 1), default: 1000 }; +const RESONANCE = numberRow('resonance', 'Resonance', 0, 1, 0.01); +const PASS_FILTERS: { value: FxPassFilter; label: string }[] = [ + { value: 'lowpass', label: 'Low-pass' }, + { value: 'highpass', label: 'High-pass' }, + { value: 'bandpass', label: 'Band-pass' }, +]; +const PASS = { + key: 'filter', + label: 'Type', + kind: 'enum' as const, + default: 'lowpass', + options: PASS_FILTERS, +}; + +function passOf(settings: PropertyBag): FxPassFilter { + return PASS_FILTERS.find((option) => option.value === settings[PASS.key])?.value ?? 'lowpass'; +} + +export const SoundFxFilter: SoundFxNodeDef = { + type: 'SoundFxFilter', + label: 'Filter', + category: 'sound', + description: + 'Takes away what is above "cutoff" (low-pass), what is below it (high-pass), or both, leaving a band around it (band-pass). "Resonance" from 0 to 1 raises what is right at the cutoff: at 0 a low- or high-pass has no bump, towards 1 it rings, and a band-pass gets narrower. In an instrument an Envelope can move both while a note plays, for a filter sweep.', + inputs: [audioIn, followingIn(CUTOFF), followingIn(RESONANCE)], + outputs: [audioOut], + params: [PASS], + emit: (sounds, amounts, settings) => + withStages(arriving(sounds), [ + { + kind: 'pass', + filter: passOf(settings), + cutoff: held(amounts.cutoff, CUTOFF), + resonance: held(amounts.resonance, RESONANCE), + }, + ]), +}; + const MIX = { ...numberRow('mix', 'Mix', 0, 1, 0.01), default: 0.5 }; const MIX_CURVE = { key: 'curve', @@ -282,15 +412,19 @@ export const SoundFxMix: SoundFxNodeDef = { category: 'sound', description: 'Blends two sounds: at 0 only a is heard, at 1 only b. With the curve "Crossfade" 0.5 is half of each. With "Both at center" each stays full up to the middle and only fades on the other half: 0.25 is a full and b half, 0.5 both full, added as two sounds on top of each other. Both ways can start at the same Input, each with its own effects, its own Pitch included.', - inputs: [{ key: 'a', label: 'a', type: 'audio' }, { key: 'b', label: 'b', type: 'audio' }, numberIn(MIX)], + inputs: [ + { key: 'a', label: 'a', type: 'audio' }, + { key: 'b', label: 'b', type: 'audio' }, + followingIn(MIX), + ], outputs: [audioOut], params: [MIX_CURVE], emit: (sounds, amounts, settings) => { const mix = held(amounts.mix, MIX); const a = sounds.a ?? EMPTY_PLAN; const b = sounds.b ?? EMPTY_PLAN; - if (mix.param === undefined && mix.value <= 0) return a; - if (mix.param === undefined && mix.value >= 1) return b; + if (isFixed(mix) && mix.value <= 0) return a; + if (isFixed(mix) && mix.value >= 1) return b; return settings[MIX_CURVE.key] === 'both' ? { mix, curve: 'both', a, b, stages: [] } : { mix, a, b, stages: [] }; @@ -357,6 +491,7 @@ export const SOUND_FX_NODES: readonly SoundFxNodeDef[] = [ SoundFxGain, SoundFxPitch, SoundFxEq, + SoundFxFilter, SoundFxMix, SoundFxDelay, ]; @@ -373,13 +508,55 @@ function delaySeconds(time: number, beats: boolean, tempo: number): number { return Math.min(DELAY_MAX, beats ? secondsOfQuarters(time, heldTempo(tempo)) : time); } +export function envelopeOf(plan: FxEnvelopePlan, params: FxParams): FxEnvelope { + return { + id: plan.id, + attack: amountOf(plan.attack, params), + decay: amountOf(plan.decay, params), + sustain: amountOf(plan.sustain, params), + release: amountOf(plan.release, params), + }; +} + +/** The Envelope moving an amount, its two ends as what the node holds (`convert`); none for a still one. */ +function modOf(amount: FxAmount, params: FxParams, convert = (value: number) => value): FxMod | undefined { + const mod = amount.mod; + if (!mod) return undefined; + return { + envelope: envelopeOf(mod.envelope, params), + from: convert(amountOf(mod.from, params)), + to: convert(amountOf(mod.to, params)), + }; +} + +const dbToGain = (db: number) => 10 ** (db / 20); + /** A stage fed by a parameter stays at 0 as well: a value handed in later has to find it there. */ function stagesOf(plan: FxPlan, params: FxParams, tempo: number): FxStage[] { const stages: FxStage[] = []; for (const stage of plan.stages) { if (stage.kind === 'gain') { const db = amountOf(stage.db, params); - if (db !== 0 || stage.db.param !== undefined) stages.push({ kind: 'gain', gain: 10 ** (db / 20) }); + const mod = modOf(stage.db, params, dbToGain); + if (db !== 0 || stage.db.param !== undefined || mod) + stages.push({ kind: 'gain', gain: dbToGain(db), ...(mod && { mod }) }); + continue; + } + if (stage.kind === 'envelope') { + stages.push({ kind: 'envelope', envelope: envelopeOf(stage.envelope, params) }); + continue; + } + if (stage.kind === 'pass') { + const cutoffMod = modOf(stage.cutoff, params); + const qMod = modOf(stage.resonance, params, (r) => passQ(stage.filter, r)); + stages.push({ + kind: 'pass', + filter: stage.filter, + cutoff: amountOf(stage.cutoff, params), + q: passQ(stage.filter, amountOf(stage.resonance, params)), + ...(cutoffMod && { cutoffMod }), + ...(qMod && { qMod }), + }); continue; } if (stage.kind === 'delay') { @@ -394,7 +571,9 @@ function stagesOf(plan: FxPlan, params: FxParams, tempo: number): FxStage[] { continue; } const gain = amountOf(stage.gain, params); - if (gain !== 0 || stage.gain.param !== undefined) stages.push({ ...stage, gain }); + const mod = modOf(stage.gain, params); + if (gain !== 0 || stage.gain.param !== undefined || mod) + stages.push({ ...stage, gain, ...(mod && { mod }) }); } return stages; } @@ -407,16 +586,24 @@ function stagesOf(plan: FxPlan, params: FxParams, tempo: number): FxStage[] { export function fxChainOf(plan: FxPlan, params: FxParams = {}, tempo = DEFAULT_TEMPO): FxChain { const stages = stagesOf(plan, params, tempo); if (!('mix' in plan)) { - const pitch = plan.pitch.reduce((sum, amount) => sum + amountOf(amount, params), 0); - const chain = { pitch, shifted: plan.pitch.length > 0, stages }; + const pitch = plan.pitch.reduce((sum, amount) => (amount.mod ? sum : sum + amountOf(amount, params)), 0); + const pitchMods = plan.pitch.flatMap((amount) => modOf(amount, params) ?? []); + const chain = { + pitch, + shifted: plan.pitch.length > 0, + stages, + ...(pitchMods.length > 0 && { pitchMods }), + }; return plan.source ? { ...chain, source: plan.source } : chain; } + const mixMod = modOf(plan.mix, params); return { mix: amountOf(plan.mix, params), ...(plan.curve === 'both' && { curve: plan.curve }), a: fxChainOf(plan.a, params, tempo), b: fxChainOf(plan.b, params, tempo), stages, + ...(mixMod && { mixMod }), }; } diff --git a/src/editor/debugHook.ts b/src/editor/debugHook.ts index f9017da1..9320c709 100644 --- a/src/editor/debugHook.ts +++ b/src/editor/debugHook.ts @@ -2,9 +2,16 @@ import type { Application } from 'pixi.js'; import { physicsStatus } from '../runtime/physics/engine'; import type { SchedulerOptions } from '../core/music/Scheduler'; import type { Sequence } from '../core/music/sequence'; -import type { FxStage } from '../core/soundfx/nodes'; +import type { FxEnvelope, FxStage } from '../core/soundfx/nodes'; import { Music } from '../viewport/music'; -import { bothAtCenter, soundLibrary, stageNodes } from '../viewport/SoundLibrary'; +import { + bothAtCenter, + envelopeDriver, + followCurve, + releaseEnvelope, + soundLibrary, + stageNodes, +} from '../viewport/SoundLibrary'; import { curveHooks } from './debug/curveHooks'; import { documentHooks } from './debug/documentHooks'; import { graphHooks } from './debug/graphHooks'; @@ -61,6 +68,15 @@ export function installDebugHook(app: Application, state: EditorState, editor: E /** What drives the two shares of a Mix "Both at center", in any context (unstarted). */ bothAtCenter: (context: BaseAudioContext, mix: number, a: GainNode, b: GainNode) => bothAtCenter(context, mix, a, b), + /** The curve of an Envelope from `at` (unstarted), its release, and how it moves a number (#335). */ + envelopeDriver: (context: BaseAudioContext, envelope: FxEnvelope, at: number) => + envelopeDriver(context, envelope, at), + releaseEnvelope: (level: AudioParam, envelope: FxEnvelope, startedAt: number, at: number) => + releaseEnvelope(level, envelope, startedAt, at), + stageWith: (context: BaseAudioContext, stage: FxStage, driver: ConstantSourceNode) => + stageNodes(context, stage, (mod, param, ratio, map) => + followCurve(context, driver, mod, param, ratio, map), + ), }, /** * The scheduler (#332), which nothing in the editor plays yet: a check hands it a sequence of diff --git a/src/editor/graph/EdgeLayer.ts b/src/editor/graph/EdgeLayer.ts index e29e7d90..22218b0f 100644 --- a/src/editor/graph/EdgeLayer.ts +++ b/src/editor/graph/EdgeLayer.ts @@ -1,5 +1,5 @@ import { Container, Graphics, type Text } from 'pixi.js'; -import { type GraphEdge, portsCompatible } from '../../core/graph'; +import { type GraphEdge, portsMeet } from '../../core/graph'; import { Theme, textStyle, truncate, crispText } from '../../ui'; import { dashedLine } from '../../viewport/symbols'; import { @@ -177,10 +177,9 @@ export class EdgeLayer extends Container { const out = from.def.outputs.find((p) => p.key === edge.from.port); const inp = to.def.inputs.find((p) => p.key === edge.to.port); const type = out?.type ?? 'any'; - const inType = inp?.type ?? 'any'; // One end on a port that is gone: the edge is drawn, but never as a working link. const ghost = out?.ghost === true || inp?.ghost === true; - const valid = !ghost && portsCompatible(type, inType); + const valid = !ghost && portsMeet({ type }, inp ?? { type: 'any' }); const color = hot ? Theme.colors.hot : valid ? portColor(type) : Theme.colors.danger; const pins = { start: { x: from.x + a.x, y: from.y + a.y }, diff --git a/src/editor/graph/NodeView.ts b/src/editor/graph/NodeView.ts index cdcc52c0..e249bd2c 100644 --- a/src/editor/graph/NodeView.ts +++ b/src/editor/graph/NodeView.ts @@ -99,6 +99,8 @@ export function portColor(type: PortType): number { return c.portFxParams; case 'trigger': return c.portTrigger; + case 'envelope': + return c.portEnvelope; case 'any': return c.portAny; } diff --git a/src/editor/graph/wiring.ts b/src/editor/graph/wiring.ts index 90edd7f1..53efb3a0 100644 --- a/src/editor/graph/wiring.ts +++ b/src/editor/graph/wiring.ts @@ -9,8 +9,8 @@ import { firstPortFor, type Graph, type PortSide, - type PortType, - portTypeOf, + type PortDef, + portOf, STATE_IN, STATE_OUT, } from '../../core/graph'; @@ -102,14 +102,14 @@ export class GraphWiring { private openPaletteForDrop(drag: { from: PortSide; pointer: Point }, global: Point): void { const graph = this.host.asset()?.graph; const from = drag.from; - const fromType = graph && portTypeOf(graph, from, graphLookup(this.host.state.project)); - if (!graph || fromType === undefined) return; + const fromPort = graph && portOf(graph, from, graphLookup(this.host.state.project)); + if (!graph || fromPort === undefined) return; const at = this.host.toGraph(global); this.pendingDrop = drag; const palette = new NodePalette( this.host.entries(), - (entry) => this.addConnectedNode(entry, from, fromType, at), - (def) => defAcceptsFrom(def, fromType, from.isOutput), + (entry) => this.addConnectedNode(entry, from, fromPort, at), + (def) => defAcceptsFrom(def, fromPort, from.isOutput), ); palette.onClose = () => { this.pendingDrop = null; @@ -119,11 +119,11 @@ export class GraphWiring { } /** Adding and wiring go in as one command, so a single undo takes the whole gesture back. */ - private addConnectedNode(entry: PaletteEntry, from: PortSide, fromType: PortType, at: Point): void { + private addConnectedNode(entry: PaletteEntry, from: PortSide, fromPort: PortDef, at: Point): void { const assetId = this.host.assetId(); if (!assetId) return; const add = this.host.newNode(entry, at); - const port = firstPortFor(entry.def, fromType, from.isOutput); + const port = firstPortFor(entry.def, fromPort, from.isOutput); const commands: Command[] = [add]; if (port) commands.push(connectSides(assetId, from, { nodeId: add.node.id, port, isOutput: !from.isOutput })); diff --git a/src/ui/Theme.ts b/src/ui/Theme.ts index 36913d00..c11f2ea2 100644 --- a/src/ui/Theme.ts +++ b/src/ui/Theme.ts @@ -82,6 +82,7 @@ export const Theme = { portAudio: 0xf0c060, portFxParams: 0xd0629b, portTrigger: 0xb5e853, + portEnvelope: 0xff7a59, portAny: 0xbdbdbd, /** Ring around the port a running edge drag would connect to. */ portTarget: 0xffffff, diff --git a/src/viewport/SoundLibrary.ts b/src/viewport/SoundLibrary.ts index df3584b7..6746b55c 100644 --- a/src/viewport/SoundLibrary.ts +++ b/src/viewport/SoundLibrary.ts @@ -1,9 +1,13 @@ import { DELAY_MAX, DRY_VOICE, + FLAT_Q, fxChainOf, type FxChain, type FxDelay, + type FxEnvelope, + type FxMod, + type FxPass, fxRings, fxTailOf, type FxParams, @@ -11,6 +15,7 @@ import { type FxStage, type FxVoice, } from '../core/soundfx/nodes'; +import { envelopeLevel, modCurve, releaseTime, SETTLE } from '../core/instrument/envelope'; import type { EaseShare } from '../core/model/animation'; import { DEFAULT_TEMPO } from '../core/music/time'; import type { AudioSink, NoteTiming } from '../runtime/systems/Audio'; @@ -56,6 +61,20 @@ interface Source { leave: (() => void) | null; } +/** Whether a note's end leaves a source as it is: a one-shot, which plays to the end of its sound. */ +const playsThrough = (source: Source) => !source.gate && !source.leave; + +/** + * One Envelope of a note (#335): what carries its curve on the audio thread, and the sources whose + * volume it is – they end with its release. One that only moves numbers has none. + */ +interface EnvelopeRun { + envelope: FxEnvelope; + driver: ConstantSourceNode; + sources: Source[]; + released: boolean; +} + /** * One sound in flight: its sources, its own gain, the file it came from and who is making it. A * voice has a source per way through its sound FX (#316), started together and as long as each @@ -69,9 +88,17 @@ interface Voice { file: string; /** The behavior this voice belongs to; `''` for one nothing owns (see `AudioSink.play`). */ owner: string; + /** When it started on the audio clock, what its Envelopes count from. */ + at: number; sources: Source[]; - /** What carries a number of it on the audio thread (a Mix "Both at center"): stopped when it goes. */ + /** Sources already set to end, by a release or a stop. */ + ending: Set; + /** + * What carries a number of it on the audio thread (a Mix "Both at center", an Envelope): stopped + * when it goes. + */ drivers: ConstantSourceNode[]; + envelopes: EnvelopeRun[]; /** Calls the end of one source in, once however it comes: by itself or by a release. */ finish: ((source: Source) => void) | null; gain: GainNode; @@ -111,6 +138,7 @@ interface Wiring { rings: boolean; sources: Source[]; drivers: ConstantSourceNode[]; + envelopes: Map; pitches: number[]; knobs: Knob[]; stages: number; @@ -249,8 +277,23 @@ export class SoundLibrary implements AudioSink { if (waiting) waiting.released = true; const voice = this.voices.find((other) => other.id === id); if (!voice) return; + this.letGo(voice, Math.max(at, this.audio().currentTime), false); + } + + /** + * Ends a note at `when`, or with `all` lets a stopped one ring (#335). Its Envelopes fall away in + * their release, and the sources whose volume they are end with it – but one under only one-shots + * holds, unless `all`. Of the rest the gated ones fade out at once, with `all` every one. + */ + private letGo(voice: Voice, when: number, all: boolean): void { + for (const run of voice.envelopes) { + if (run.released) continue; + if (!all && run.sources.length > 0 && run.sources.every(playsThrough)) continue; + run.released = true; + releaseEnvelope(run.driver.offset, run.envelope, voice.at, when); + this.endAt(voice, run.sources, when + releaseTime(run.envelope)); + } const context = this.audio(); - const when = Math.max(at, context.currentTime); const leaving = voice.sources.filter((source) => !source.gate && source.leave); const leave = () => { for (const source of leaving) source.leave!(); @@ -258,12 +301,24 @@ export class SoundLibrary implements AudioSink { if (leaving.length > 0 && when > context.currentTime) this.after(when, () => this.voices.includes(voice) && leave()); else leave(); - const gated = voice.sources.filter((source) => source.gate); - for (const source of gated) if (source.fade) fadeOut(source.fade.gain, when); - if (gated.length === 0) return; - this.after(when + FADE, () => { + this.endAt( + voice, + voice.sources.filter((source) => all || source.gate), + when, + ); + } + + /** Fades these sources of a voice out from `at` and lets them go then, each only once. */ + private endAt(voice: Voice, sources: readonly Source[], at: number): void { + const going = sources.filter((source) => !voice.ending.has(source)); + if (going.length === 0) return; + for (const source of going) { + voice.ending.add(source); + if (source.fade) fadeOut(source.fade.gain, at); + } + this.after(at + FADE, () => { if (!this.voices.includes(voice)) return; - for (const source of gated) { + for (const source of going) { source.stop(); voice.finish?.(source); } @@ -502,11 +557,16 @@ export class SoundLibrary implements AudioSink { /** * Ends these voices now: `onended` only fires next frame and the counts have to be right here. A * voice is faded out rather than cut, so it does not click. With `letRing` one with echoes only - * loses its sound and stays until they have faded (#324); one already ringing is left to it. + * loses its sound and stays until they have faded (#324); one already ringing is left to it. A + * note with Envelopes falls away in their release first (#335). */ private silence(going: Voice[], letRing = false): void { this.stopped += going.filter((voice) => !(letRing && voice.ringing)).length; for (const voice of going) { + if (letRing && voice.envelopes.length > 0) { + this.letGo(voice, this.audio().currentTime, true); + continue; + } if (letRing && voice.rings) { if (!voice.ringing) this.ringOut(voice); continue; @@ -622,6 +682,7 @@ export class SoundLibrary implements AudioSink { rings, sources: [], drivers: [], + envelopes: new Map(), pitches: [], knobs: [], stages: 0, @@ -638,13 +699,17 @@ export class SoundLibrary implements AudioSink { stages: wiring.stages, duration: Math.max(0, ...files.map((name) => this.buffers.get(name)?.duration ?? 0)), at: when, + envelopes: wiring.envelopes.size, }; const voice: Voice = { id, file, owner, + at: when, sources, + ending: new Set(), drivers: wiring.drivers, + envelopes: [...wiring.envelopes.values()], finish: null, gain, fx, @@ -672,26 +737,38 @@ export class SoundLibrary implements AudioSink { */ private wire(wiring: Wiring, chain: FxChain): AudioNode { const { context, knobs } = wiring; + const follow = this.follower(wiring); + const first = wiring.sources.length; let tail: AudioNode; if ('mix' in chain) { tail = context.createGain(); const both = chain.curve === 'both'; + // An Envelope moving the share stands in for the knob: a parameter no longer reaches it. + const moved = chain.mixMod; const wayA = this.wire(wiring, chain.a); - const shareA = gainNode(context, both ? 0 : 1 - chain.mix); + const shareA = gainNode(context, both || moved ? 0 : 1 - chain.mix); wayA.connect(shareA).connect(tail); - if (!both) knobs.push({ param: shareA.gain, target: 1 - chain.mix, even: 'value' }); + if (!both) { + knobs.push({ param: moved ? null : shareA.gain, target: 1 - chain.mix, even: 'value' }); + if (moved) follow(moved, shareA.gain, false, (mix) => 1 - mix); + } const wayB = this.wire(wiring, chain.b); - const shareB = gainNode(context, both ? 0 : chain.mix); + const shareB = gainNode(context, both || moved ? 0 : chain.mix); wayB.connect(shareB).connect(tail); - if (!both) knobs.push({ param: shareB.gain, target: chain.mix, even: 'value' }); - else { - const driver = bothAtCenter(context, chain.mix, shareA, shareB); - knobs.push({ param: driver.offset, target: chain.mix, even: 'value' }); + if (!both) { + knobs.push({ param: moved ? null : shareB.gain, target: chain.mix, even: 'value' }); + if (moved) follow(moved, shareB.gain, false); + } else { + const driver = bothAtCenter(context, moved ? 0 : chain.mix, shareA, shareB); + knobs.push({ param: moved ? null : driver.offset, target: chain.mix, even: 'value' }); + if (moved) follow(moved, driver.offset, false); wiring.drivers.push(driver); } } else tail = this.source(wiring, chain); + const under = wiring.sources.slice(first); for (const stage of chain.stages) { - const made = stageNodes(context, stage); + if (stage.kind === 'envelope') this.envelope(wiring, stage.envelope).sources.push(...under); + const made = stageNodes(context, stage, follow); tail.connect(made.input); tail = made.output; knobs.push(...made.knobs); @@ -700,6 +777,23 @@ export class SoundLibrary implements AudioSink { return tail; } + /** Sends the curve of an Envelope into a number of a voice, shaped to what that number holds (#335). */ + private follower(wiring: Wiring): Follow { + return (mod, param, ratio, map) => + followCurve(wiring.context, this.envelope(wiring, mod.envelope).driver, mod, param, ratio, map); + } + + /** The run of an Envelope in a voice, started on its first use: one curve however much it moves. */ + private envelope(wiring: Wiring, envelope: FxEnvelope): EnvelopeRun { + const known = wiring.envelopes.get(envelope.id); + if (known) return known; + const driver = envelopeDriver(wiring.context, envelope, wiring.at); + wiring.drivers.push(driver); + const run: EnvelopeRun = { envelope, driver, sources: [], released: false }; + wiring.envelopes.set(envelope.id, run); + return run; + } + /** * A way's sound: the pitch worklet where the way has a Pitch, which keeps the length whatever the * pitch does (#317), and otherwise the buffer as it is. Without the worklet a pitched way plays at @@ -715,7 +809,7 @@ export class SoundLibrary implements AudioSink { return sound.connect(fade); } - private sound(wiring: Wiring, { pitch, shifted, source: own }: Way): AudioNode { + private sound(wiring: Wiring, { pitch, shifted, source: own, pitchMods }: Way): AudioNode { const { context } = wiring; const file = own?.file ?? wiring.file; const loop = own ? own.loop : wiring.loop; @@ -758,7 +852,11 @@ export class SoundLibrary implements AudioSink { gate, leave: own && loop ? () => node.port.postMessage('leave') : null, }); - wiring.knobs.push({ param: node.parameters.get('pitch')!, target: pitch, even: 'value' }); + const knob = node.parameters.get('pitch')!; + wiring.knobs.push({ param: knob, target: pitch, even: 'value' }); + // On top of the pitch the way has anyway, which a parameter can still move. + const follow = this.follower(wiring); + for (const mod of pitchMods ?? []) follow(mod, knob, false); return node; } const source = context.createBufferSource(); @@ -850,6 +948,8 @@ interface LastVoice { duration: number; /** When it started on the audio clock, what a check times it from. */ at: number; + /** How many Envelopes shape it (#335). */ + envelopes: number; } /** Whether any way of a plan has a Pitch, and so needs the worklet. */ @@ -888,30 +988,127 @@ interface StageNodes { knobs: Knob[]; } -/** Exported for a check that renders a stage offline in a real browser (test hook). */ -export function stageNodes(context: BaseAudioContext, stage: FxStage): StageNodes { +/** + * Moves `param` along the curve of an Envelope (#335), from `from` to `to` in equal steps or equal + * ratios, `map` turning that into what the node holds. The param's own value is added, so one that + * follows alone is set to 0 first. + */ +export type Follow = (mod: FxMod, param: AudioParam, ratio: boolean, map?: (value: number) => number) => void; + +/** + * Shapes the curve an Envelope's `driver` carries into `param` through a WaveShaper. Exported for a + * check that renders it offline in a real browser (test hook). + */ +export function followCurve( + context: BaseAudioContext, + driver: AudioNode, + mod: FxMod, + param: AudioParam, + ratio: boolean, + map?: (value: number) => number, +): void { + const shape = context.createWaveShaper(); + shape.curve = modCurve(mod, ratio, map); + driver.connect(shape).connect(param); +} + +/** + * Exported for a check that renders a stage offline in a real browser (test hook). Without `follow` + * a number an Envelope moves stays at 0, and an Envelope's own stage silent. + */ +export function stageNodes(context: BaseAudioContext, stage: FxStage, follow?: Follow): StageNodes { if (stage.kind === 'gain') { - const gain = gainNode(context, stage.gain); - return { input: gain, output: gain, knobs: [{ param: gain.gain, target: stage.gain, even: 'ratio' }] }; + const gain = gainNode(context, stage.mod ? 0 : stage.gain); + if (stage.mod) follow?.(stage.mod, gain.gain, true); + const knob: Knob = { param: stage.mod ? null : gain.gain, target: stage.gain, even: 'ratio' }; + return { input: gain, output: gain, knobs: [knob] }; } + if (stage.kind === 'envelope') { + const gain = gainNode(context, 0); + follow?.({ envelope: stage.envelope, from: 0, to: 1 }, gain.gain, false); + return { input: gain, output: gain, knobs: [] }; + } + if (stage.kind === 'pass') return passNodes(context, stage, follow); if (stage.kind === 'delay') return delayNodes(context, stage); const filter = context.createBiquadFilter(); filter.type = stage.filter; filter.frequency.value = stage.frequency; - filter.gain.value = stage.gain; + filter.gain.value = stage.mod ? 0 : stage.gain; filter.Q.value = stage.q; + if (stage.mod) follow?.(stage.mod, filter.gain, false); return { input: filter, output: filter, - knobs: [{ param: filter.gain, target: stage.gain, even: 'value' }], + knobs: [{ param: stage.mod ? null : filter.gain, target: stage.gain, even: 'value' }], + }; +} + +/** A Filter (#335): a cutoff that moves in octaves, and a Q – a band-pass's as a factor, so in ratios too. */ +function passNodes(context: BaseAudioContext, stage: FxPass, follow?: Follow): StageNodes { + const filter = context.createBiquadFilter(); + filter.type = stage.filter; + filter.frequency.value = stage.cutoffMod ? 0 : stage.cutoff; + filter.Q.value = stage.qMod ? 0 : stage.q; + const qEven = stage.filter === 'bandpass' ? 'ratio' : 'value'; + if (stage.cutoffMod) follow?.(stage.cutoffMod, filter.frequency, true); + if (stage.qMod) follow?.(stage.qMod, filter.Q, qEven === 'ratio'); + return { + input: filter, + output: filter, + knobs: [ + { param: stage.cutoffMod ? null : filter.frequency, target: stage.cutoff, even: 'ratio' }, + { param: stage.qMod ? null : filter.Q, target: stage.q, even: qEven }, + ], }; } /** - * The Q of a low-pass with no bump at its cutoff, in the decibels Web Audio takes it in. An echo - * coming out of the damping louder than it went in would grow with every round. + * What carries the curve of an Envelope (#335) from `at`: a straight line up to 1, then easing down + * towards the sustain. Its caller starts it and stops it with the voice. Exported for a check that + * renders it offline in a real browser (test hook). */ -const FLAT_Q = 20 * Math.log10(Math.SQRT1_2); +export function envelopeDriver( + context: BaseAudioContext, + envelope: FxEnvelope, + at: number, +): ConstantSourceNode { + const driver = context.createConstantSource(); + const level = driver.offset; + level.value = 0; + const top = at + envelope.attack; + if (envelope.attack > 0) { + level.setValueAtTime(0, at); + level.linearRampToValueAtTime(1, top); + } else level.setValueAtTime(1, at); + if (envelope.sustain < 1) { + if (envelope.decay > 0) level.setTargetAtTime(envelope.sustain, top, envelope.decay / SETTLE); + else level.setValueAtTime(envelope.sustain, top); + } + return driver; +} + +/** + * Lets an Envelope that started at `startedAt` fall away from `at`, from the level it has by then – + * mid-attack too – to silence in its release. Exported for the same check (test hook). + */ +export function releaseEnvelope( + level: AudioParam, + envelope: FxEnvelope, + startedAt: number, + at: number, +): void { + const held = level as Omit & + Partial>; + if (held.cancelAndHoldAtTime) held.cancelAndHoldAtTime(at); + else { + // Firefox: the level by then is known, the curve up to it is what was planned. + const reached = envelopeLevel(envelope, at - startedAt); + level.cancelScheduledValues(at); + if (at - startedAt < envelope.attack) level.linearRampToValueAtTime(reached, at); + else level.setValueAtTime(reached, at); + } + level.setTargetAtTime(0, at, releaseTime(envelope) / SETTLE); +} /** * An echo (#327): the sound goes on as it is, and through a delay line whose output is damped, fed @@ -1062,7 +1259,8 @@ function levelsOf(chain: FxChain, into: number[] = []): number[] { } else if (chain.shifted) into.push(chain.pitch); for (const stage of chain.stages) if (stage.kind === 'delay') into.push(stage.time, stage.feedback, stage.damping, stage.mix); - else into.push(stage.gain); + else if (stage.kind === 'pass') into.push(stage.cutoff, stage.q); + else if (stage.kind !== 'envelope') into.push(stage.gain); return into; } diff --git a/tests/core/envelope.test.ts b/tests/core/envelope.test.ts new file mode 100644 index 00000000..3de512c2 --- /dev/null +++ b/tests/core/envelope.test.ts @@ -0,0 +1,232 @@ +/** + * An Envelope (#335) shapes a note of an instrument: on the way of the sound it is its volume, and its + * curve can move a number on the audio thread – the cutoff of a Filter, a Pitch, a Gain, a Mix, a + * band of the EQ – from "from" at rest to "to" at the top. + */ +import { describe, expect, it } from 'vitest'; +import { CreateAssetCommand, CreateSoundCommand } from '../../src/core/commands/assetCommands'; +import { ConnectCommand } from '../../src/core/commands/graphCommands'; +import { createEmptyGraph, type GraphNode } from '../../src/core/graph/Graph'; +import { portsMeet } from '../../src/core/graph/NodeRegistry'; +import { compileInstrument, instrumentSourcesOf } from '../../src/core/instrument/compile'; +import { envelopeLevel, modCurve, releaseTime, SETTLE } from '../../src/core/instrument/envelope'; +import { VELOCITY_PARAM } from '../../src/core/instrument/nodes'; +import type { BehaviorAsset, InstrumentAsset } from '../../src/core/model/Asset'; +import { compileSoundFx } from '../../src/core/soundfx/compile'; +import { FLAT_Q, fxChainOf, type FxEnvelope, type FxPlan, passQ } from '../../src/core/soundfx/nodes'; +import { graphEditor, makeProject } from '../helpers'; + +const FILE = `${'c3'.repeat(32)}.wav`; + +/** An instrument with nothing in it, a Notes node and a Sample playing `FILE` on every note. */ +function makeInstrument() { + const f = makeProject(); + const command = new CreateAssetCommand('instrument', 'Pad'); + f.stack.execute(command); + const instrument = command.asset as InstrumentAsset; + instrument.graph = createEmptyGraph(); + const made = new CreateSoundCommand('Pad', FILE); + f.stack.execute(made); + const edit = graphEditor(f, instrument as unknown as BehaviorAsset); + const notes = edit.addNode('InstrumentNotes'); + const sample = edit.addNode('InstrumentSample', { sound: made.asset.id }); + const output = edit.addNode('InstrumentOutput'); + edit.connect(notes, 'any', sample, 'trigger'); + const compile = () => compileInstrument(instrument.graph, 60, instrumentSourcesOf(f.project)); + return { ...f, instrument, edit, notes, sample, output, compile }; +} + +const ENVELOPE: FxEnvelope = { id: 'e', attack: 0.1, decay: 0.2, sustain: 0.5, release: 0.3 }; + +describe('the curve of an Envelope', () => { + it('rises in a straight line, then falls towards the sustain, 60 dB of the way there by the end of the decay', () => { + expect(envelopeLevel(ENVELOPE, 0)).toBe(0); + expect(envelopeLevel(ENVELOPE, 0.05)).toBeCloseTo(0.5, 9); + expect(envelopeLevel(ENVELOPE, 0.1)).toBeCloseTo(1, 9); + expect(envelopeLevel(ENVELOPE, 0.3)).toBeCloseTo(0.5 + 0.5 / 1000, 9); + expect(envelopeLevel(ENVELOPE, 10)).toBeCloseTo(0.5, 9); + // Without an attack it starts at the top, without a decay it is at the sustain right after. + expect(envelopeLevel({ ...ENVELOPE, attack: 0 }, 0.000001)).toBeCloseTo(1, 3); + expect(envelopeLevel({ ...ENVELOPE, decay: 0 }, 0.1)).toBe(0.5); + expect(Math.exp(-SETTLE)).toBeCloseTo(0.001, 9); + }); + + it('takes 10 ms to release at least, so a note does not click', () => { + expect(releaseTime(ENVELOPE)).toBe(0.3); + expect(releaseTime({ ...ENVELOPE, release: 0 })).toBe(0.01); + }); + + it('moves a number evenly, or in equal ratios, and below 0 stays where it rests', () => { + const at = (curve: Float32Array, s: number) => curve[Math.round(((s + 1) / 2) * (curve.length - 1))]!; + const even = modCurve({ envelope: ENVELOPE, from: 2, to: 10 }, false); + expect([at(even, -1), at(even, 0), at(even, 0.5), at(even, 1)]).toEqual([2, 2, 6, 10]); + const ratio = modCurve({ envelope: ENVELOPE, from: 200, to: 5000 }, true); + expect(at(ratio, 0.5)).toBeCloseTo(1000, 1); + // A ratio needs both ends above 0; otherwise it goes evenly. + expect(at(modCurve({ envelope: ENVELOPE, from: 0, to: 1 }, true), 0.5)).toBe(0.5); + const mapped = modCurve({ envelope: ENVELOPE, from: 0, to: 1 }, false, (mix) => 1 - mix); + expect([at(mapped, 0), at(mapped, 1)]).toEqual([1, 0]); + }); +}); + +describe('a Filter', () => { + it('has no bump at resonance 0 and rings towards 1; a band-pass gets narrower', () => { + expect(passQ('lowpass', 0)).toBeCloseTo(FLAT_Q, 9); + expect(passQ('highpass', 1)).toBe(20); + expect(passQ('bandpass', 0)).toBeCloseTo(0.7, 9); + expect(passQ('bandpass', 1)).toBeCloseTo(20, 9); + expect(passQ('lowpass', 7)).toBe(20); + }); + + it('is an effect of a sound FX as well, its cutoff in Hz and its resonance as a Q', () => { + const f = makeProject(); + const made = new CreateAssetCommand('soundfx', 'Muffle'); + f.stack.execute(made); + const fx = made.asset as unknown as BehaviorAsset; + const edit = graphEditor(f, fx); + const input = edit.addNode('SoundFxInput'); + const filter = edit.addNode('SoundFxFilter', { filter: 'highpass', cutoff: 300, resonance: 1 }); + const output = edit.addNode('SoundFxOutput'); + edit.wire([ + [input, 'out', filter, 'in'], + [filter, 'out', output, 'in'], + ]); + const plan = compileSoundFx((fx as unknown as { graph: never }).graph).compiled!; + expect(fxChainOf(plan).stages).toEqual([{ kind: 'pass', filter: 'highpass', cutoff: 300, q: 20 }]); + }); +}); + +describe('an Envelope in an instrument', () => { + it('on the way of the sound is its volume, its numbers held to their rows', () => { + const f = makeInstrument(); + const envelope = f.edit.addNode('InstrumentEnvelope', { attack: 0.02, sustain: 3, release: -1 }); + f.edit.wire([ + [f.sample, 'out', envelope, 'in'], + [envelope, 'out', f.output, 'in'], + ]); + const build = f.compile(); + expect(build.reason).toBe(''); + expect(fxChainOf(build.layers[0]!).stages).toEqual([ + { kind: 'envelope', envelope: { id: envelope.id, attack: 0.02, decay: 0.1, sustain: 1, release: 0 } }, + ]); + }); + + it('moves the cutoff of a Filter from "from" to "to", and its resonance as a Q', () => { + const f = makeInstrument(); + const envelope = f.edit.addNode('InstrumentEnvelope', { from: 200, to: 5000 }); + const sweep = f.edit.addNode('InstrumentEnvelope', { decay: 1, from: 0, to: 1 }); + const filter = f.edit.addNode('SoundFxFilter'); + f.edit.wire([ + [f.sample, 'out', filter, 'in'], + [envelope, 'value', filter, 'cutoff'], + [sweep, 'value', filter, 'resonance'], + [filter, 'out', f.output, 'in'], + ]); + const build = f.compile(); + // An Envelope moving numbers only shapes no sound, and nothing is missing for it. + expect(build.problems).toEqual({}); + const [stage] = fxChainOf(build.layers[0]!).stages; + expect(stage).toMatchObject({ + kind: 'pass', + filter: 'lowpass', + cutoff: 200, + cutoffMod: { envelope: { id: envelope.id }, from: 200, to: 5000 }, + qMod: { envelope: { id: sweep.id, decay: 1 }, from: FLAT_Q, to: 20 }, + }); + }); + + it('holds both ends to the row of the number it moves, and fills in what a note hands it', () => { + const f = makeInstrument(); + const envelope = f.edit.addNode('InstrumentEnvelope', { from: 0, to: 30000 }); + const filter = f.edit.addNode('SoundFxFilter'); + f.edit.wire([ + [f.sample, 'out', filter, 'in'], + [envelope, 'value', filter, 'cutoff'], + [f.notes, 'velocity', envelope, 'sustain'], + [filter, 'out', f.output, 'in'], + ]); + const chain = fxChainOf(f.compile().layers[0]!, { [VELOCITY_PARAM]: 0.25 }); + expect(chain.stages[0]).toMatchObject({ + cutoff: 20, + cutoffMod: { from: 20, to: 20000, envelope: { sustain: 0.25 } }, + }); + }); + + it('moves a Pitch on top of the one the way has, a Gain in its factor, a band and a Mix', () => { + const f = makeInstrument(); + const envelope = f.edit.addNode('InstrumentEnvelope', { from: -12, to: 0 }); + const pitch = f.edit.addNode('SoundFxPitch'); + const gain = f.edit.addNode('SoundFxGain'); + const eq = f.edit.addNode('SoundFxEq'); + const mix = f.edit.addNode('SoundFxMix'); + f.edit.wire([ + [f.sample, 'out', pitch, 'in'], + [envelope, 'value', pitch, 'semitones'], + [pitch, 'out', gain, 'in'], + [envelope, 'value', gain, 'gain'], + [gain, 'out', eq, 'in'], + [envelope, 'value', eq, 'mid'], + [eq, 'out', mix, 'a'], + [f.sample, 'out', mix, 'b'], + [envelope, 'value', mix, 'mix'], + [mix, 'out', f.output, 'in'], + ]); + const chain = fxChainOf(f.compile().layers[0]!); + if (!('mix' in chain) || 'mix' in chain.a) throw new Error('a Mix of two ways'); + // The mix rests at "from", held to 0..1; at 0 it is not taken for "only a". + expect(chain.mix).toBe(0); + expect(chain.mixMod).toMatchObject({ from: 0, to: 0 }); + expect(chain.a.pitch).toBe(0); + expect(chain.a.shifted).toBe(true); + expect(chain.a.pitchMods).toMatchObject([{ from: -12, to: 0 }]); + expect(chain.a.stages).toMatchObject([ + { kind: 'gain', mod: { from: 10 ** (-12 / 20), to: 1 } }, + { kind: 'filter', frequency: 1000, mod: { from: -12, to: 0 } }, + ]); + }); + + it('cannot move a number the audio thread does not hold, like the time of a Delay', () => { + const f = makeInstrument(); + const envelope = f.edit.addNode('InstrumentEnvelope'); + const delay = f.edit.addNode('SoundFxDelay'); + const value = { type: 'envelope' as const }; + expect(portsMeet(value, { type: 'number', followsEnvelope: true })).toBe(true); + expect(portsMeet(value, { type: 'number' })).toBe(false); + expect(portsMeet(value, { type: 'any' })).toBe(false); + expect(portsMeet({ type: 'number' }, { type: 'envelope' })).toBe(false); + expect(() => + f.stack.execute( + new ConnectCommand( + f.instrument.id, + { node: envelope.id, port: 'value' }, + { node: delay.id, port: 'time' }, + ), + ), + ).toThrow('Cannot connect envelope to number'); + // Saved that way anyhow, it says so rather than playing it. + f.instrument.graph.edges.push({ + id: 'by-hand', + from: { node: envelope.id, port: 'value' }, + to: { node: delay.id, port: 'time' }, + }); + f.edit.wire([ + [f.sample, 'out', delay, 'in'], + [delay, 'out', f.output, 'in'], + ]); + expect(f.compile().problems[delay.id]).toBe('"Time" cannot follow an Envelope'); + }); + + it('with its sound way connected on but nothing coming in, says so', () => { + const f = makeInstrument(); + const envelope: GraphNode = f.edit.addNode('InstrumentEnvelope'); + f.edit.connect(envelope, 'out', f.output, 'in'); + expect(f.compile().problems[envelope.id]).toBe('Nothing comes in: connect it to a Sample'); + }); + + it('leaves the plan of a note without one as it was', () => { + const f = makeInstrument(); + f.edit.connect(f.sample, 'out', f.output, 'in'); + const plan: FxPlan = f.compile().layers[0]!; + expect(plan.stages).toEqual([]); + }); +}); diff --git a/tests/core/soundfx.test.ts b/tests/core/soundfx.test.ts index 496ca4b3..03f93157 100644 --- a/tests/core/soundfx.test.ts +++ b/tests/core/soundfx.test.ts @@ -118,12 +118,20 @@ describe('the sound FX asset', () => { 'SoundFxGain', 'SoundFxPitch', 'SoundFxEq', + 'SoundFxFilter', 'SoundFxMix', 'SoundFxDelay', ]); f.edit.addNode('SoundFxInput'); f.edit.addNode('SoundFxOutput'); - expect(ids()).toEqual(['SoundFxGain', 'SoundFxPitch', 'SoundFxEq', 'SoundFxMix', 'SoundFxDelay']); + expect(ids()).toEqual([ + 'SoundFxGain', + 'SoundFxPitch', + 'SoundFxEq', + 'SoundFxFilter', + 'SoundFxMix', + 'SoundFxDelay', + ]); const behavior = makeBehavior(f, 'Logic'); expect(paletteEntries(f.project, behavior).some((e) => e.def.category === 'sound')).toBe(false); diff --git a/tests/viewport/sound-library.test.ts b/tests/viewport/sound-library.test.ts index 21f99d44..dd707298 100644 --- a/tests/viewport/sound-library.test.ts +++ b/tests/viewport/sound-library.test.ts @@ -6,7 +6,7 @@ * The device is stubbed – nothing makes a noise in a unit test, and Web Audio does not exist here. */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { FxPlan, FxStage, FxStagePlan, FxVoice } from '../../src/core/soundfx/nodes'; +import type { FxEnvelopePlan, FxPlan, FxStage, FxStagePlan, FxVoice } from '../../src/core/soundfx/nodes'; import type { PitchOptions } from '../../src/viewport/pitchWorklet'; import { SoundLibrary } from '../../src/viewport/SoundLibrary'; @@ -307,6 +307,7 @@ function planOf(chain: Chain): FxPlan { const stages = chain.stages.map((stage): FxStagePlan => { if (stage.kind === 'gain') return { kind: 'gain', db: { value: 20 * Math.log10(stage.gain) } }; if (stage.kind === 'filter') return { ...stage, gain: { value: stage.gain } }; + if (stage.kind !== 'delay') throw new Error(`no plan for a ${stage.kind} here`); return { ...stage, time: { value: stage.time }, @@ -412,6 +413,7 @@ describe('the sound library', () => { stages: 0, duration: 1.5, at: 0, + envelopes: 0, }); }); @@ -1164,3 +1166,152 @@ describe('notes of an instrument (#331)', () => { expect(library.playNote('instrument:i', [{ pitch: [], stages: [] }], {}, 1, '', 0)).toBe(0); }); }); + +describe('notes with an Envelope (#335)', () => { + const PLANNED: FxEnvelopePlan = { + id: 'env', + attack: { value: 0.1 }, + decay: { value: 0.2 }, + sustain: { value: 0.5 }, + release: { value: 0.3 }, + }; + + /** A Sample's way, under the Envelope where `shaped`, with more stages after it. */ + const way = (gate: boolean, shaped = true, stages: FxStagePlan[] = []): FxPlan => ({ + pitch: [], + stages: [...(shaped ? [{ kind: 'envelope' as const, envelope: PLANNED }] : []), ...stages], + source: { file: FILE, volume: 0.5, loop: false, loopStart: 0, loopEnd: 0, gate }, + }); + + async function note(library: SoundLibrary, plan: FxPlan, duration = 0): Promise { + const voice = library.playNote('instrument:i', [plan], {}, 1, '', duration); + await settle(); + await arrive(); + return voice; + } + + /** What carries the curve: the first silent source made, before any timer. */ + const driver = () => clocks[0]!; + + it('starts its curve with the note: straight up in the attack, then easing towards the sustain', async () => { + const library = new SoundLibrary(); + await note(library, way(true)); + expect(library.stats().last?.envelopes).toBe(1); + expect(driver().started).toBe(true); + expect(driver().offset.ramps).toEqual([['linear', 1, 0.1]]); + expect(driver().offset.glides).toEqual([0.5]); + // Voice gain, the Sample's volume, and the Envelope's gain, which is all the curve. + expect(gains.map((gain) => gain.value)).toEqual([1, 0.5, 0]); + expect(wires).toContain('clock > shaper'); + expect([shapers[0]!.curve![512], shapers[0]!.curve![1024]]).toEqual([0, 1]); + }); + + it('a gated source plays on through the release, and the note ends with it', async () => { + const library = new SoundLibrary(); + const voice = await note(library, way(true)); + library.release(voice); + // Released at 0, in the attack still: from 0, falling to silence in 0.3 s. + expect(driver().offset.ramps).toEqual([ + ['linear', 1, 0.1], + ['linear', 0, 0], + ]); + expect(driver().offset.glides).toEqual([0.5, 0]); + expect(gains[1]!.ramps).toEqual([ + ['set', 0.5, 0.3], + ['linear', 0, 0.31], + ]); + clockTo(0.3); + expect(sources[0]!.stopped).toBe(false); + expect(library.playing(voice)).toBe(true); + clockTo(); + expect(sources[0]!.stopped).toBe(true); + expect(library.playing(voice)).toBe(false); + expect(driver().stopped).toBe(true); + }); + + it('a note ending mid-decay releases from the level its curve has by then', async () => { + const library = new SoundLibrary(); + await note(library, way(true), 0.2); + const level = 0.5 + 0.5 * Math.exp((-0.1 * Math.log(1000)) / 0.2); + expect(driver().offset.ramps).toEqual([ + ['linear', 1, 0.1], + ['set', level, 0.2], + ]); + expect(gains[1]!.ramps).toEqual([ + ['set', 0.5, 0.5], + ['linear', 0, 0.51], + ]); + }); + + it('a one-shot under an Envelope alone plays through, its curve held', async () => { + const library = new SoundLibrary(); + const voice = await note(library, way(false)); + library.release(voice); + expect(driver().offset.glides).toEqual([0.5]); + clockTo(); + expect(sources[0]!.stopped).toBe(false); + sources[0]!.onended?.(); + expect(library.playing(voice)).toBe(false); + }); + + it('Stop sound lets it fall away in its release with "let ring", and cuts it without', async () => { + const library = new SoundLibrary(); + const ringing = await note(library, way(false)); + library.stopVoice(ringing, true); + expect(driver().offset.glides).toEqual([0.5, 0]); + expect(gains[1]!.ramps).toEqual([ + ['set', 0.5, 0.3], + ['linear', 0, 0.31], + ]); + expect(gains[0]!.ramps).toEqual([]); + expect(library.stats().stopped).toBe(1); + clockTo(); + expect(library.playing(ringing)).toBe(false); + + const cut = await note(library, way(true)); + library.stopVoice(cut); + expect(gains[3]!.ramps).toEqual([['linear', 0, 0.01]]); + expect(library.playing(cut)).toBe(false); + }); + + it('one that only moves a cutoff ends nothing: the gated source fades with the note as before', async () => { + const library = new SoundLibrary(); + const sweep: FxStagePlan = { + kind: 'pass', + filter: 'lowpass', + cutoff: { value: 200, mod: { envelope: PLANNED, from: { value: 200 }, to: { value: 5000 } } }, + resonance: { value: 0 }, + }; + const voice = await note(library, way(true, false, [sweep])); + // The cutoff is all the curve; the Q stays as it is. + expect(filters[0]!.frequency.value).toBe(0); + expect(filters[0]!.Q.value).toBeCloseTo(20 * Math.log10(Math.SQRT1_2), 9); + const curve = shapers[0]!.curve!; + expect([curve[0], curve[512], curve[768], curve[1024]]).toEqual([ + 200, + 200, + expect.closeTo(1000, 1), + expect.closeTo(5000, 1), + ]); + library.release(voice); + expect(gains[1]!.ramps).toEqual([['linear', 0, 0.01]]); + expect(driver().offset.glides).toEqual([0.5, 0]); + // A parameter does not reach a number an Envelope moves. + expect(library.levels('instrument:i')).toEqual([[200, expect.closeTo(-3.01, 2)]]); + }); + + it('moves a pitch on top of the one its way has, through the worklet', async () => { + const library = new SoundLibrary(); + const plan: FxPlan = { + ...way(true, false), + pitch: [ + { value: 3 }, + { value: -12, mod: { envelope: PLANNED, from: { value: -12 }, to: { value: 0 } } }, + ], + }; + await note(library, plan); + expect(worklets[0]!.pitch.value).toBe(3); + expect(wires).toContain('shaper > ?'); + expect([shapers[0]!.curve![512], shapers[0]!.curve![1024]]).toEqual([-12, 0]); + }); +}); diff --git a/tools/e2e/34-sound.mjs b/tools/e2e/34-sound.mjs index 084f61fa..9607b76d 100644 --- a/tools/e2e/34-sound.mjs +++ b/tools/e2e/34-sound.mjs @@ -845,7 +845,7 @@ export async function sound(t) { const insPalette = await json('__gm.overlayTexts()'); check( 'its palette offers its own nodes and the effects, but no Input and no behavior node', - ['Notes', 'Sample', 'Parameters', 'Sound FX', 'Gain', 'Delay'].every((label) => + ['Notes', 'Sample', 'Parameters', 'Sound FX', 'Envelope', 'Gain', 'Filter', 'Delay'].every((label) => insPalette.includes(label), ) && !insPalette.includes('Input') && @@ -1051,6 +1051,76 @@ export async function sound(t) { JSON.stringify(onsets), ); + // #335: an Envelope's curve as Chrome renders it – up in the attack, easing towards the sustain, + // and from wherever it is when the note ends down to silence in the release, mid-attack too – + // and a Gain it moves through a WaveShaper, from a quarter to full in equal ratios. + const curves = await evaluate(` + (async () => { + const RATE = 48000; + const envelope = { id: 'e', attack: 0.1, decay: 0.2, sustain: 0.5, release: 0.3 }; + const render = async (releaseAt, moved) => { + const context = new OfflineAudioContext(1, RATE, RATE); + const driver = __gm.sound.envelopeDriver(context, envelope, 0); + if (releaseAt !== null) __gm.sound.releaseEnvelope(driver.offset, envelope, 0, releaseAt); + if (moved) { + const steady = context.createConstantSource(); + const stage = { kind: 'gain', gain: 1, mod: { envelope, from: 0.25, to: 1 } }; + const made = __gm.sound.stageWith(context, stage, driver); + steady.connect(made.input); + made.output.connect(context.destination); + steady.start(); + } else driver.connect(context.destination); + driver.start(); + const out = (await context.startRendering()).getChannelData(0); + return (s) => Math.round(out[Math.round(s * RATE)] * 10000) / 10000; + }; + const held = await render(0.5, false); + const early = await render(0.05, false); + const gain = await render(null, true); + return { + held: [0.05, 0.1, 0.3, 0.5, 0.65, 0.8].map(held), + early: [0.05, 0.15, 0.35].map(early), + gain: [0.05, 0.1, 0.9].map(gain), + }; + })()`); + const closeTo = (values, expected) => values.every((v, i) => Math.abs(v - expected[i]) < 0.002); + check( + 'an Envelope rises, eases to its sustain and releases from where it is, and moves a Gain with that curve', + closeTo(curves.held, [0.5, 1, 0.5005, 0.5, 0.0158, 0.0005]) && + closeTo(curves.early, [0.5, 0.05, 0.0005]) && + closeTo(curves.gain, [0.5, 1, 0.5]), + JSON.stringify(curves), + ); + + // A note of 0.2 s through an Envelope with a release of 1 s: the looped blip plays on through the + // release, and the voice goes with it – without one it would end 10 ms after the note. + await mutate(` + const graph = __gm.state.project.assets.find((a) => a.id === ${JSON.stringify(insId)}).graph; + const into = graph.edges.find((e) => e.to.node === graph.nodes.find((n) => n.type === 'InstrumentOutput').id); + graph.nodes.push(${JSON.stringify(node('ins-env', 'InstrumentEnvelope', 300, 120, { release: 1 }))}); + graph.edges.push({ id: 'ins-e-env', from: { node: 'ins-env', port: 'out' }, to: { ...into.to } }); + into.to = { node: 'ins-env', port: 'in' };`); + await evaluate('__gm.sound.stopAll()'); + const beforeShaped = await stats(); + await evaluate( + `__gm.music.play(${JSON.stringify({ notes: [{ tick: 0, length: 240, note: 60, velocity: 1, instrument: insId }] })}, { tempo: 150 })`, + ); + const shaped = await waitFor(noteNow, (s) => s.stats.started > beforeShaped.started); + const ringOut = await trace(noteNow, (s) => s.stats.playing === 0); + await evaluate('__gm.music.stop()'); + const shapedAt = shaped.stats.last?.at ?? 0; + const shapedGone = +(ringOut.at(-1).clock - shapedAt).toFixed(3); + // 0.2 s of note and 1 s of release; up to the 150 ms waitFor polls at on top. + check( + 'a note through an Envelope plays on through its release and goes with it', + shaped.stats.last?.envelopes === 1 && + ringOut.some((s) => s.stats.playing === 1 && s.clock - shapedAt > 0.8) && + ringOut.at(-1).stats.playing === 0 && + shapedGone > 1.15 && + shapedGone < 1.6, + JSON.stringify({ last: shaped.stats.last, shapedGone }), + ); + await mutate(` const graph = __gm.state.project.assets.find((a) => a.id === 'sn-logic').graph; graph.nodes = graph.nodes.filter((n) => n.id !== 'sn-note');