diff --git a/README.md b/README.md index ed1e00c9..e84a30fa 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh The browser suite and the three example games need a machine with a GPU: ```bash -node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 1919 checks (GM_STEPS=20 for one step) +node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 1921 checks (GM_STEPS=20 for one step) # every tool signs in first – see the Accounts page node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs diff --git a/src/core/graph/NodeRegistry.ts b/src/core/graph/NodeRegistry.ts index 888a3244..7389d629 100644 --- a/src/core/graph/NodeRegistry.ts +++ b/src/core/graph/NodeRegistry.ts @@ -111,6 +111,21 @@ export interface DerivedShape { inOrder?: true; } +/** + * A small picture a node draws below its rows (#375), as wide as the node and `height` tall. Its + * shapes are flat point lists `[x0, y0, x1, y1, …]` in that box, y down: a `guide` is a faint line, an + * `area` is filled, a `line` is drawn strong. + */ +export interface NodePicture { + height: number; + shapes(params: PropertyBag, w: number): PictureShape[]; +} + +export interface PictureShape { + kind: 'guide' | 'area' | 'line'; + points: number[]; +} + /** Palette, port rendering, connection validation and inspector fields are all derived from this. */ export interface NodeDef { type: string; @@ -135,6 +150,8 @@ export interface NodeDef { idleParams?(params: PropertyBag, fed?: (key: string) => boolean): string[]; /** Whether the node does nothing at this value set and is drawn faded, like a bypassed effect (#354). */ dimmed?(params: PropertyBag): boolean; + /** What the node shows of its settings below its rows, drawn rather than written (#375). */ + picture?: NodePicture; /** * Which side the ports this node declares for itself (`GraphNode.ports`) appear on. Setting it * is what gives a node the port editor in the inspector. `both`: every port is an output, and an diff --git a/src/core/instrument/compile.ts b/src/core/instrument/compile.ts index 177f701e..063a0a9a 100644 --- a/src/core/instrument/compile.ts +++ b/src/core/instrument/compile.ts @@ -35,8 +35,10 @@ import { type SoundFxNodeDef, type SoundFxProject, SoundFxReverb, + tapId, withPitch, withStages, + withTaps, } from '../soundfx/nodes'; import { CHIP_ROWS, @@ -129,9 +131,15 @@ export function instrumentParams(graph: Graph): readonly DeclaredPort[] { * One note of an instrument (#331): the graph walked back from the Output, with the triggers that * stand for this note on and every other one off. A source whose trigger is off is silence, and so * is everything it alone feeds; a Mix with one silent side is its other side. `note` and `frequency` - * are numbers here, `velocity` and the parameters stay open for every voice to fill in. + * are numbers here, `velocity` and the parameters stay open for every voice to fill in. `assetId` is + * the instrument, which the taps of a node are named by (#374). */ -export function compileInstrument(graph: Graph, note: number, sources: InstrumentSources): InstrumentBuild { +export function compileInstrument( + graph: Graph, + note: number, + sources: InstrumentSources, + assetId = '', +): InstrumentBuild { const project: SoundFxProject = { soundFile: (assetId) => sources.sound(assetId)?.file ?? '' }; const walk = graphEmitter(graph, { isDef: isInstrumentDef, @@ -230,9 +238,9 @@ export function compileInstrument(graph: Graph, note: number, sources: Instrumen return { out: { plans: [def.emit({ a, b }, amounts, node.params, project)] } }; } const [arriving] = plansAt('in'); - return { - out: arriving ? { plans: [def.emit({ in: arriving }, amounts, node.params, project)] } : SILENT, - }; + if (!arriving) return { out: SILENT }; + const plan = def.emit({ in: arriving }, amounts, node.params, project); + return { out: { plans: [def.tapped ? withTaps(arriving, plan, tapId(assetId, node.id)) : plan] } }; } /** A Sample's sound, or null after saying why it has none – also where its trigger is off. */ @@ -416,7 +424,7 @@ export function instrumentBuildFor( } let build = cached.notes.get(note); if (!build) { - build = compileInstrument(asset.graph, note, sources); + build = compileInstrument(asset.graph, note, sources, asset.id); cached.notes.set(note, build); } return build; diff --git a/src/core/soundfx/compile.ts b/src/core/soundfx/compile.ts index ce004eb4..f0c2ae25 100644 --- a/src/core/soundfx/compile.ts +++ b/src/core/soundfx/compile.ts @@ -17,6 +17,8 @@ import { SOUND_FX_OUTPUT, type SoundFxNodeDef, type SoundFxProject, + tapId, + withTaps, } from './nodes'; export interface SoundFxBuild { @@ -49,9 +51,10 @@ function inputOutputs(node: GraphNode): Record { * A sound FX graph to the way a voice is sent: walked back from the Output to the Input, every node * on the way adding what it does. An input takes one edge, so only a Mix joins two ways into one * (#316); a node fed from nowhere is a problem rather than silence. A number fed from a parameter - * stays open in the plan until a voice is started (#311). + * stays open in the plan until a voice is started (#311). `assetId` is the graph's asset, which the + * taps of a node are named by (#374). */ -export function compileSoundFx(graph: Graph, project?: SoundFxProject): SoundFxBuild { +export function compileSoundFx(graph: Graph, project?: SoundFxProject, assetId = ''): SoundFxBuild { const walk = graphEmitter(graph, { isDef: isSoundFxNodeDef, noun: 'sound FX', @@ -64,7 +67,9 @@ export function compileSoundFx(graph: Graph, project?: SoundFxProject): SoundFxB for (const [key, value] of Object.entries(inputs)) if ('amount' in value) amounts[key] = value.amount; else sounds[key] = value.plan; - return { out: { plan: def.emit(sounds, amounts, node.params, project) } }; + const plan = def.emit(sounds, amounts, node.params, project); + if (!def.tapped) return { out: { plan } }; + return { out: { plan: withTaps(sounds.in ?? EMPTY_PLAN, plan, tapId(assetId, node.id)) } }; }, unfed: (node, port) => { if (port.type === 'number') return { amount: { value: asNumber(inlineValue(node, port)) } }; @@ -124,6 +129,7 @@ export function soundFxBuildFor(asset: SoundFxAsset, project?: Project): SoundFx const build = compileSoundFx( asset.graph, project && { soundFile: (assetId) => playedSoundById(project, assetId) }, + asset.id, ); builds.set(asset.id, { fingerprint, build }); return build; diff --git a/src/core/soundfx/nodes.ts b/src/core/soundfx/nodes.ts index e6785138..73266515 100644 --- a/src/core/soundfx/nodes.ts +++ b/src/core/soundfx/nodes.ts @@ -16,7 +16,7 @@ import { toneCutoff, } from './saturation'; import { PHASER_STAGES } from './phaser'; -import { eqBandsOf, eqStagesOf, hasGain, isEqBandHeard, MAX_EQ_BANDS } from './parametricEq'; +import { EQ_PICTURE, eqBandsOf, eqStagesOf, hasGain, isEqBandHeard, MAX_EQ_BANDS } from './parametricEq'; export { SOUND_FX_INPUT, SOUND_FX_OUTPUT }; @@ -48,7 +48,18 @@ export type FxStage = | FxLfo<'tremolo'> | FxLfo<'vibrato'> | FxReverb - | FxWet; + | FxWet + | FxTap; + +/** + * Where the sound in front of a node or behind it can be listened to (#374); nothing is done to it. + * `id` names the node in the project (`tapId`), `at` the side. + */ +export interface FxTap { + kind: 'tap'; + id: string; + at: 'in' | 'out'; +} /** * What one node adds, blended with what arrives at it (#354): `wet` of its stages and the rest of the @@ -319,7 +330,8 @@ export type FxStagePlan = | { kind: 'tremolo'; shape: LfoShape; rate: FxAmount; depth: FxAmount } | { kind: 'vibrato'; shape: LfoShape; rate: FxAmount; depth: FxAmount } | ({ kind: 'reverb'; impulse?: string } & Record<'decay' | 'damping' | 'predelay' | 'mix', FxAmount>) - | { kind: 'wet'; wet: FxAmount; stages: readonly FxStagePlan[] }; + | { kind: 'wet'; wet: FxAmount; stages: readonly FxStagePlan[] } + | FxTap; /** A chain with its parameters still open: compiled once, filled in for every voice by `fxChainOf`. */ export type FxPlan = @@ -350,6 +362,8 @@ export const DRY_VOICE: FxVoice = { plan: EMPTY_PLAN, params: {} }; */ export interface SoundFxNodeDef extends NodeDef { category: 'sound'; + /** The sound in front of the node and behind it can be listened to (#374), see `withTaps`. */ + tapped?: true; emit( sounds: Readonly>, amounts: Readonly>, @@ -417,6 +431,26 @@ export function withStages(plan: FxPlan, stages: readonly FxStagePlan[]): FxPlan return stages.length === 0 ? plan : { ...plan, stages: [...plan.stages, ...stages] }; } +/** A node by the asset its graph is in: node ids alone repeat in a copied asset. */ +export const tapId = (assetId: string, nodeId: string): string => `${assetId}:${nodeId}`; + +/** + * What a tapped node made of `before`, with a tap where its own stages begin and one behind them + * (#374): the sound as it came in, and as it goes on – blended by Wet, or not touched when bypassed. + */ +export function withTaps(before: FxPlan, after: FxPlan, id: string): FxPlan { + const at = before.stages.length; + return { + ...after, + stages: [ + ...after.stages.slice(0, at), + { kind: 'tap', id, at: 'in' }, + ...after.stages.slice(at), + { kind: 'tap', id, at: 'out' }, + ], + }; +} + const BYPASS = { key: 'bypass', label: 'Bypass', kind: 'boolean' as const, default: false }; const WET = { ...numberRow('wet', 'Wet', 0, 1, 0.01), default: 1 }; @@ -596,10 +630,12 @@ export const SoundFxParametricEq: SoundFxNodeDef = effect( type: 'SoundFxParametricEq', label: 'Parametric EQ', category: 'sound', - description: `Up to ${MAX_EQ_BANDS} filters working together, drawn as one curve. Double-click the node to open its editor: double-click the curve to add a band, drag a band for its frequency and gain, turn the mouse wheel over it for its Q. Each band is a peak, a low or high shelf, a low or high cut, a band-pass or a notch, and a cut, a band-pass or a notch can be steeper with its slope.${BLENDS}`, + description: `Up to ${MAX_EQ_BANDS} filters working together, drawn as one curve, here in small. Double-click the node to open its editor: double-click the curve to add a band, drag a band for its frequency and gain, turn the mouse wheel over it for its Q. Each band is a peak, a low or high shelf, a low or high cut, a band-pass or a notch, and a cut, a band-pass or a notch can be steeper with its slope. While the editor is seen, the spectrum of what passes the node is drawn behind the curve: faint as it comes in, strong as it goes on.${BLENDS}`, inputs: [audioIn], outputs: [audioOut], params: [], + picture: EQ_PICTURE, + tapped: true, emit: (sounds, _amounts, settings) => withStages( arriving(sounds), @@ -1287,6 +1323,10 @@ const dbToGain = (db: number) => 10 ** (db / 20); function stagesOf(plan: readonly FxStagePlan[], params: FxParams, tempo: number): FxStage[] { const stages: FxStage[] = []; for (const stage of plan) { + if (stage.kind === 'tap') { + stages.push(stage); + continue; + } if (stage.kind === 'wet') { const wetMod = modOf(stage.wet, params); stages.push({ diff --git a/src/core/soundfx/parametricEq.ts b/src/core/soundfx/parametricEq.ts index 4875874a..cfad2682 100644 --- a/src/core/soundfx/parametricEq.ts +++ b/src/core/soundfx/parametricEq.ts @@ -3,6 +3,7 @@ * node, and the curve they make together – worked out as Web Audio's BiquadFilterNode does, so the * editor draws what is heard. */ +import type { NodePicture, PictureShape } from '../graph/NodeRegistry'; import { isList, type PropertyBag, type PropertyValue, type StructValue } from '../model/Properties'; import type { FxPassFilter } from './nodes'; @@ -224,3 +225,89 @@ export function eqCurves( }); return { bands: each, sum }; } + +/** + * A picture of the curve, in the EQ's tab and in its node: frequency to the right in octaves, from + * 20 Hz to 20 kHz, and dB up, the same distance either side of 0. + */ +export interface EqArea { + w: number; + h: number; +} + +/** The dB a picture of the curve shows above and below 0. */ +const EQ_VIEW_DB = EQ_GAIN.max; + +const SPAN = Math.log(EQ_FREQUENCY.max / EQ_FREQUENCY.min); + +export const frequencyToX = (frequency: number, area: EqArea): number => + (Math.log(frequency / EQ_FREQUENCY.min) / SPAN) * area.w; + +export const xToFrequency = (x: number, area: EqArea): number => + EQ_FREQUENCY.min * Math.exp((Math.max(0, Math.min(area.w, x)) / area.w) * SPAN); + +export const dbToY = (db: number, area: EqArea): number => area.h / 2 - (db / EQ_VIEW_DB) * (area.h / 2); + +export const yToDb = (y: number, area: EqArea): number => ((area.h / 2 - y) / (area.h / 2)) * EQ_VIEW_DB; + +/** + * The curves of `eqCurves` laid onto an area through `steps` + 1 points, evenly in octaves. A point + * stays within `overflow` of the area, so a deep cut runs out of sight rather than down to -300 dB. + */ +export function eqCurvePoints( + bands: readonly EqBand[], + area: EqArea, + steps: number, + overflow = 0, +): { xs: number[]; bands: number[][]; sum: number[] } { + const xs = Array.from({ length: steps + 1 }, (_, i) => (i / steps) * area.w); + const yOf = (db: number) => Math.max(-overflow, Math.min(area.h + overflow, dbToY(db, area))); + const curves = eqCurves( + bands, + xs.map((x) => xToFrequency(x, area)), + ); + return { xs, bands: curves.bands.map((dbs) => dbs.map(yOf)), sum: curves.sum.map(yOf) }; +} + +/** + * The area between a curve and the 0 dB line at `zero`, as flat point lists: one shape for every + * stretch on one side of the line. A single shape would cross itself where the curve crosses the + * line, and its fill would come apart. + */ +export function fillShapes(xs: readonly number[], ys: readonly number[], zero: number): number[][] { + const shapes: number[][] = []; + let shape = [xs[0]!, zero]; + for (let i = 0; i < xs.length; i++) { + const after = ys[i]! - zero; + const before = i > 0 ? ys[i - 1]! - zero : 0; + if (before * after < 0) { + const x = xs[i - 1]! + ((xs[i]! - xs[i - 1]!) * before) / (before - after); + shapes.push([...shape, x, zero]); + shape = [x, zero]; + } + shape.push(xs[i]!, ys[i]!); + } + shapes.push([...shape, xs.at(-1)!, zero]); + return shapes; +} + +/** How tall the curve is in the node, and through how many points it is drawn there (#375). */ +const PICTURE_HEIGHT = 60; +const PICTURE_STEPS = 96; + +/** The sum of the bands in the node (#375): the 0 dB line, the area to the curve, and the curve. */ +export function eqPictureShapes(bands: readonly EqBand[], area: EqArea): PictureShape[] { + const { xs, sum } = eqCurvePoints(bands, area, PICTURE_STEPS); + const zero = dbToY(0, area); + return [ + { kind: 'guide', points: [0, zero, area.w, zero] }, + ...fillShapes(xs, sum, zero).map((points) => ({ kind: 'area' as const, points })), + { kind: 'line', points: xs.flatMap((x, i) => [x, sum[i]!]) }, + ]; +} + +/** Without a band the box stays, with its 0 dB line: the node does not jump at the first one. */ +export const EQ_PICTURE: NodePicture = { + height: PICTURE_HEIGHT, + shapes: (params, w) => eqPictureShapes(eqBandsOf(params), { w, h: PICTURE_HEIGHT }), +}; diff --git a/src/editor/Editor.ts b/src/editor/Editor.ts index a6c05d03..075bdb34 100644 --- a/src/editor/Editor.ts +++ b/src/editor/Editor.ts @@ -1299,6 +1299,8 @@ export class Editor implements PlayHost { if (this.shows('song')) this.songEditor.tick(); else this.songEditor.hideMeters(); if (this.shows('sound')) this.soundEditor.tick(); + if (this.shows('eq')) this.eqEditor.tick(); + else this.eqEditor.hideSpectrum(); // Play repaints the machine tab itself (see `PlaySession.tick`); an editor run has no session // to do it, and its states move just the same. if (this.shows('statemachine') && this.editorRun.isRunning) this.stateMachineEditor.canvas.redraw(); @@ -1334,7 +1336,8 @@ export class Editor implements PlayHost { (this.shows('curve') && this.curveEditor.isPlaying) || (this.shows('pattern') && this.patternEditor.isPlaying) || (this.shows('song') && this.songEditor.isPlaying) || - (this.shows('sound') && this.soundEditor.isPlaying) + (this.shows('sound') && this.soundEditor.isPlaying) || + (this.shows('eq') && this.eqEditor.hasSpectrum) ); } diff --git a/src/editor/debug/graphHooks.ts b/src/editor/debug/graphHooks.ts index bd034def..55dd8e1f 100644 --- a/src/editor/debug/graphHooks.ts +++ b/src/editor/debug/graphHooks.ts @@ -1,4 +1,5 @@ import { NameTransitionCommand } from '../../core/commands/graphCommands'; +import { soundLibrary } from '../../viewport/SoundLibrary'; import { currentNodePalette } from '../graph/NodePalette'; import { rectOf } from '../textProbe'; import { centreOf, snapButton, type HookContext } from './hookHelpers'; @@ -111,6 +112,8 @@ export function graphHooks({ state, editor }: HookContext) { }), playAt: () => centreOf(editor.soundFxEditor.playButton), stopAt: () => centreOf(editor.soundFxEditor.stopButton), + /** As the header's Play does, also while the tab is behind another one. */ + play: () => editor.soundFxEditor.play(), /** The header's parameter values, and setting one as its field would. */ params: () => editor.soundFxEditor.previewValues, setParam: (key: string, value: number) => editor.soundFxEditor.setPreviewValue(key, value), @@ -134,6 +137,12 @@ export function graphHooks({ state, editor }: HookContext) { }; }, typeAt: () => centreOf(editor.eqEditor.typeButton), + /** + * The spectra drawn behind the curve (#374): how many, the loudest bin in front of the node and + * behind it in the last one, and the taps – which node is listened to, how many play, how many + * of them feed an analyser. + */ + spectrum: () => ({ ...editor.eqEditor.curve.seen, taps: soundLibrary.spectrumTaps() }), }, instrument: { current: () => editor.instrumentEditor.currentAssetId, diff --git a/src/editor/graph/NodeView.ts b/src/editor/graph/NodeView.ts index f870994c..d0303147 100644 --- a/src/editor/graph/NodeView.ts +++ b/src/editor/graph/NodeView.ts @@ -11,6 +11,7 @@ import { type NodeCategory, type NodeDef, nodeDef, + type NodePicture, type PortDef, type PortType, type Value, @@ -125,6 +126,8 @@ export interface PortHit { /** Draws one graph node entirely from its NodeDef; positioned in graph coordinates by GraphCanvas. */ export class NodeView extends Widget { private readonly box = new Graphics(); + /** What the node draws of its settings below its rows (#375). */ + private readonly picture = new Graphics(); private readonly ports = new Graphics(); private readonly texts: Text[] = []; /** The input and output text of each port row, among `texts`. */ @@ -159,6 +162,7 @@ export class NodeView extends Widget { */ private textsOf = ''; private drawnOf = ''; + private pictureOf = ''; /** How often the texts and the box were really made, for the checks. */ readonly made = { texts: 0, box: 0 }; @@ -178,8 +182,9 @@ export class NodeView extends Widget { // Pixi draws a shape of more than 200 vertices on its own, and the round pins are: a few hundred // nodes were a few hundred draw calls. In the batch they share one. this.box.context.batchMode = 'batch'; + this.picture.context.batchMode = 'batch'; this.ports.context.batchMode = 'batch'; - this.addChild(this.box, this.ports); + this.addChild(this.box, this.picture, this.ports); this.sync(node, { nodes: [node], edges: [] }, null, def); } @@ -250,7 +255,10 @@ export class NodeView extends Widget { inputs: graph.edges.some((e) => e.to.node === node.id), outputs: graph.edges.some((e) => e.from.node === node.id), }; - const height = this.collapsed ? TITLE_HEIGHT : this.headerHeight() + this.rowCount() * ROW_HEIGHT + PAD; + const picture = this.shownPicture(); + const height = this.collapsed + ? TITLE_HEIGHT + : this.rowsEnd() + PAD + (picture ? picture.height + PAD / 2 : 0); // Sized before the texts: output labels are placed against the right edge. this.resize(this.collapsed ? this.collapsedWidth() : NODE_WIDTH, height); const texts = this.textSignature(); @@ -387,6 +395,47 @@ export class NodeView extends Widget { return TITLE_HEIGHT + PAD + lines * ROW_HEIGHT; } + /** Below the last port row. */ + private rowsEnd(): number { + return this.headerHeight() + this.rowCount() * ROW_HEIGHT; + } + + /** The node's picture, while the node is open. */ + private shownPicture(): NodePicture | null { + return (!this.collapsed && this.def.picture) || null; + } + + /** Where the picture sits, in node units. */ + private pictureBox(): { x: number; y: number; w: number; h: number } | null { + const picture = this.shownPicture(); + return picture && { x: PAD, y: this.rowsEnd() + PAD / 2, w: this.w - PAD * 2, h: picture.height }; + } + + /** Drawn again when the settings or the size have changed: a drag in the EQ's tab changes them. */ + private drawPicture(): void { + const picture = this.shownPicture(); + const box = this.pictureBox(); + const drawn = picture && box ? JSON.stringify([box, this.node.params]) : ''; + if (drawn === this.pictureOf) return; + this.pictureOf = drawn; + this.picture.clear(); + if (!picture || !box) return; + const c = Theme.colors; + this.picture.position.set(box.x, box.y); + this.picture.rect(0, 0, box.w, box.h).fill(c.graphBackground).stroke({ color: c.nodeBorder, width: 1 }); + for (const { kind, points } of picture.shapes(this.node.params, box.w)) { + if (kind === 'area') { + this.picture.poly(points).fill({ color: c.accent, alpha: 0.18 }); + continue; + } + this.picture.moveTo(points[0]!, points[1]!); + for (let i = 2; i < points.length; i += 2) this.picture.lineTo(points[i]!, points[i + 1]!); + this.picture.stroke( + kind === 'guide' ? { color: c.textMuted, width: 1, alpha: 0.5 } : { color: c.accent, width: 1.5 }, + ); + } + } + /** Local position of a port's centre. */ portPosition(key: string, isOutput: boolean): { x: number; y: number } | null { const list = isOutput ? this.def.outputs : this.def.inputs; @@ -490,6 +539,7 @@ export class NodeView extends Widget { } override redraw(): void { + this.drawPicture(); const drawn = this.drawSignature(); if (drawn === this.drawnOf) return; this.drawnOf = drawn; diff --git a/src/editor/soundfx/EqEditor.ts b/src/editor/soundfx/EqEditor.ts index 30b3e614..111cd765 100644 --- a/src/editor/soundfx/EqEditor.ts +++ b/src/editor/soundfx/EqEditor.ts @@ -2,7 +2,8 @@ * The editor of a Parametric EQ node (#367), in a tab of its own (#370). The curve is the sum of the * bands, worked out as Web Audio plays them (`parametricEq.ts`); a band is dragged for frequency and * gain, wheeled for its Q, and its numbers can be typed below the curve. A drag or a scrub is one - * undo step. + * undo step. While the tab is seen, the spectrum of what passes the node is drawn behind the curve + * (#374). */ import { type FederatedPointerEvent, type FederatedWheelEvent, Graphics, type Text } from 'pixi.js'; import { SetNodeParamCommand } from '../../core/commands/graphCommands'; @@ -10,6 +11,8 @@ import type { GraphNode } from '../../core/graph'; import { findGraphAsset, type Project } from '../../core/model/Project'; import type { PropertyValue } from '../../core/model/Properties'; import { + dbToY, + type EqArea, type EqBand, EQ_BAND_TYPES, EQ_BANDS_PARAM, @@ -19,15 +22,21 @@ import { EQ_SLOPES, eqBandsOf, eqBandsValue, - eqCurves, + eqCurvePoints, + fillShapes, + frequencyToX, hasGain, hasQ, hasSlope, MAX_EQ_BANDS, newEqBand, withBandSet, + xToFrequency, + yToDb, } from '../../core/soundfx/parametricEq'; -import { SoundFxParametricEq } from '../../core/soundfx/nodes'; +import { SoundFxParametricEq, tapId } from '../../core/soundfx/nodes'; +import { soundLibrary } from '../../viewport/SoundLibrary'; +import type { Spectrum } from '../../viewport/spectrum'; import { crispText, Dropdown, @@ -47,16 +56,13 @@ import type { EditorState } from '../state'; import { TabHeader } from '../TabHeader'; import { bandAt, - dbToY, - type EqArea, EQ_GRID_DB, EQ_GRID_FREQUENCIES, - fillShapes, - frequencyToX, handleOf, + SPECTRUM_FLOOR_DB, + spectrumPeak, + spectrumYs, wheeledQ, - xToFrequency, - yToDb, } from './eqView'; /** The least the curve shrinks to; otherwise it takes the tab. */ @@ -71,6 +77,8 @@ const CURVE_STEPS = 320; const BAND_COLORS = [ 0xf2c84b, 0x6fcf97, 0x56ccf2, 0xe57373, 0xb066ff, 0xf2a75c, 0xff9ad5, 0x9b8cff, 0x3fb8c9, 0xd4c25a, ]; +/** Pixels between the points the spectrum is drawn through (#374). */ +const SPECTRUM_STEP = 2; /** What the curve asks of the tab around it. */ interface EqHost { @@ -85,18 +93,24 @@ interface EqHost { class EqCurve extends Widget { private readonly bg = new Graphics(); private readonly grid = new Graphics(); + /** What passes the node, behind the curve (#374): faint as it comes in, strong as it goes on. */ + private readonly spectrum = new Graphics(); private readonly lines = new Graphics(); private readonly handles = new Graphics(); private readonly labels: Text[] = []; private readonly numbers: Text[] = []; private drag: { index: number; startX: number; startY: number; moved: boolean } | null = null; + /** The spectrum drawn now; null while nothing has passed the node or it has fallen away. */ + private heard: Spectrum | null = null; + /** How many spectra were drawn, and the loudest bin on either side of the last one (checks). */ + seen = { frames: 0, peakIn: 0, peakOut: 0, levelOut: -Infinity, binHz: 0 }; constructor(private readonly host: EqHost) { super(); this.minWidth = MIN_CURVE_WIDTH; this.minHeight = MIN_CURVE_HEIGHT; this.cursor = 'crosshair'; - this.addChild(this.bg, this.grid, this.lines, this.handles); + this.addChild(this.bg, this.grid, this.spectrum, this.lines, this.handles); const muted = textStyle('small').clone(); muted.fill = Theme.colors.textMuted; const onHandle = textStyle('small').clone(); @@ -231,20 +245,15 @@ class EqCurve extends Widget { const bands = this.host.bands(); this.lines.clear(); - const xs = Array.from({ length: CURVE_STEPS + 1 }, (_, i) => (i / CURVE_STEPS) * this.w); - const yOf = (db: number) => Math.max(-2, Math.min(this.h + 2, dbToY(db, area))); - const curves = eqCurves( - bands, - xs.map((x) => xToFrequency(x, area)), - ); - curves.bands.forEach((dbs, i) => { - this.trace(xs, dbs.map(yOf)); + const curves = eqCurvePoints(bands, area, CURVE_STEPS, 2); + const xs = curves.xs; + curves.bands.forEach((ys, i) => { + trace(this.lines, xs, ys); this.lines.stroke({ color: BAND_COLORS[i]!, width: 1, alpha: i === this.host.picked ? 0.9 : 0.4 }); }); - const ys = curves.sum.map(yOf); - for (const shape of fillShapes(xs, ys, zero)) + for (const shape of fillShapes(xs, curves.sum, zero)) this.lines.poly(shape).fill({ color: c.accent, alpha: 0.12 }); - this.trace(xs, ys); + trace(this.lines, xs, curves.sum); this.lines.stroke({ color: c.accent, width: 2 }); this.handles.clear(); @@ -258,13 +267,58 @@ class EqCurve extends Widget { this.handles.circle(at.x, at.y, HANDLE_RADIUS + 3).stroke({ color: c.symbol, width: 2 }); text.position.set(Math.round(at.x - text.width / 2), Math.round(at.y - text.height / 2)); }); + this.drawSpectrum(); } - /** A path through the points. */ - private trace(xs: readonly number[], ys: readonly number[]): void { - this.lines.moveTo(xs[0]!, ys[0]!); - for (let i = 1; i < xs.length; i++) this.lines.lineTo(xs[i]!, ys[i]!); + /** The spectrum a frame on (#374); one all below the floor of its scale counts as none. */ + showSpectrum(spectrum: Spectrum | null): void { + const peakIn = spectrum && spectrumPeak(spectrum.in, spectrum.binHz); + const peakOut = spectrum && spectrumPeak(spectrum.out, spectrum.binHz); + const loud = Math.max(peakIn?.db ?? -Infinity, peakOut?.db ?? -Infinity) > SPECTRUM_FLOOR_DB; + const shown = loud ? spectrum : null; + if (!shown && !this.heard) return; + this.heard = shown; + if (shown) + this.seen = { + frames: this.seen.frames + 1, + peakIn: peakIn?.hz ?? 0, + peakOut: peakOut?.hz ?? 0, + levelOut: peakOut?.db ?? -Infinity, + binHz: shown.binHz, + }; + this.drawSpectrum(); } + + get hasSpectrum(): boolean { + return this.heard !== null; + } + + private drawSpectrum(): void { + this.spectrum.clear(); + const heard = this.heard; + if (!heard) return; + const c = Theme.colors; + const count = Math.floor(this.w / SPECTRUM_STEP); + const xs = Array.from({ length: count + 1 }, (_, i) => (i / count) * this.w); + for (const [dbs, color, alpha, line] of [ + [heard.in, c.textMuted, 0.12, false], + [heard.out, c.portAudio, 0.18, true], + ] as const) { + const ys = spectrumYs(dbs, heard.binHz, xs, this.area); + this.spectrum + .poly([0, this.h, ...xs.flatMap((x, i) => [x, ys[i]!]), this.w, this.h]) + .fill({ color, alpha }); + if (!line) continue; + trace(this.spectrum, xs, ys); + this.spectrum.stroke({ color, width: 1, alpha: 0.6 }); + } + } +} + +/** A path through the points. */ +function trace(g: Graphics, xs: readonly number[], ys: readonly number[]): void { + g.moveTo(xs[0]!, ys[0]!); + for (let i = 1; i < xs.length; i++) g.lineTo(xs[i]!, ys[i]!); } /** @@ -359,6 +413,7 @@ export class EqEditor extends VStack implements EqHost { show(assetId: string, nodeId: string): void { if (assetId !== this.assetId || nodeId !== this.nodeId) { this.perFrame.cancel(); + this.curve.showSpectrum(null); this.before = null; this.assetId = assetId; this.nodeId = nodeId; @@ -382,6 +437,24 @@ export class EqEditor extends VStack implements EqHost { return this.nodeId ? { assetId: this.assetId, nodeId: this.nodeId } : null; } + /** Only called while the tab is seen: the spectrum of what passes the node, a frame on (#374). */ + tick(): void { + if (!this.nodeId) return; + soundLibrary.listen(tapId(this.assetId, this.nodeId)); + this.curve.showSpectrum(soundLibrary.spectrum()); + } + + /** The tab is not seen: no analyser is fed for it. */ + hideSpectrum(): void { + soundLibrary.unlisten(); + this.curve.showSpectrum(null); + } + + /** Whether the spectrum is moving, so the tab has to be drawn every frame. */ + get hasSpectrum(): boolean { + return this.curve.hasSpectrum; + } + private node(): GraphNode | null { return eqNodeOf(this.state.project, this.assetId, this.nodeId); } diff --git a/src/editor/soundfx/eqView.ts b/src/editor/soundfx/eqView.ts index 10d80a1e..ce09f3ac 100644 --- a/src/editor/soundfx/eqView.ts +++ b/src/editor/soundfx/eqView.ts @@ -1,28 +1,16 @@ /** - * How the Parametric EQ editor (#367) lays its curve onto a picture: frequency to the right in - * octaves, from 20 Hz to 20 kHz, and dB up, the same distance either side of 0. + * What the Parametric EQ's tab (#367) lays onto its picture beside the curve, whose axes are in + * `parametricEq.ts`: the handles of the bands, and the spectrum of what passes the node (#374). */ -import { type EqBand, EQ_FREQUENCY, EQ_GAIN, EQ_Q, hasGain } from '../../core/soundfx/parametricEq'; - -export interface EqArea { - w: number; - h: number; -} - -/** The dB the picture shows above and below 0. */ -const EQ_VIEW_DB = EQ_GAIN.max; - -const SPAN = Math.log(EQ_FREQUENCY.max / EQ_FREQUENCY.min); - -export const frequencyToX = (frequency: number, area: EqArea): number => - (Math.log(frequency / EQ_FREQUENCY.min) / SPAN) * area.w; - -export const xToFrequency = (x: number, area: EqArea): number => - EQ_FREQUENCY.min * Math.exp((Math.max(0, Math.min(area.w, x)) / area.w) * SPAN); - -export const dbToY = (db: number, area: EqArea): number => area.h / 2 - (db / EQ_VIEW_DB) * (area.h / 2); - -export const yToDb = (y: number, area: EqArea): number => ((area.h / 2 - y) / (area.h / 2)) * EQ_VIEW_DB; +import { + type EqArea, + type EqBand, + dbToY, + EQ_Q, + frequencyToX, + hasGain, + xToFrequency, +} from '../../core/soundfx/parametricEq'; /** Where a band's handle sits: at its frequency, and at its gain or – without one – on the 0 dB line. */ export function handleOf(band: EqBand, area: EqArea): { x: number; y: number } { @@ -50,28 +38,6 @@ export function wheeledQ(q: number, deltaY: number): number { return Math.max(EQ_Q.min, Math.min(EQ_Q.max, Math.round(turned * 1000) / 1000)); } -/** - * The area between a curve and the 0 dB line at `zero`, as flat point lists: one shape for every - * stretch on one side of the line. A single shape would cross itself where the curve crosses the - * line, and its fill would come apart. - */ -export function fillShapes(xs: readonly number[], ys: readonly number[], zero: number): number[][] { - const shapes: number[][] = []; - let shape = [xs[0]!, zero]; - for (let i = 0; i < xs.length; i++) { - const after = ys[i]! - zero; - const before = i > 0 ? ys[i - 1]! - zero : 0; - if (before * after < 0) { - const x = xs[i - 1]! + ((xs[i]! - xs[i - 1]!) * before) / (before - after); - shapes.push([...shape, x, zero]); - shape = [x, zero]; - } - shape.push(xs[i]!, ys[i]!); - } - shapes.push([...shape, xs.at(-1)!, zero]); - return shapes; -} - /** The lines the picture is ruled with, and what they say. */ export const EQ_GRID_FREQUENCIES: readonly { hz: number; label: string }[] = [ { hz: 20, label: '20' }, @@ -86,3 +52,47 @@ export const EQ_GRID_FREQUENCIES: readonly { hz: number; label: string }[] = [ { hz: 20000, label: '20k' }, ]; export const EQ_GRID_DB: readonly number[] = [-18, -12, -6, 0, 6, 12, 18]; + +/** The spectrum's own scale (#374): full scale at the top of the picture, this many dB at its foot. */ +export const SPECTRUM_FLOOR_DB = -96; + +const spectrumY = (db: number, area: EqArea): number => + (Math.min(0, Math.max(SPECTRUM_FLOOR_DB, db)) / SPECTRUM_FLOOR_DB) * area.h; + +/** + * A spectrum, one level in dB every `binHz`, laid onto the picture at `xs`: each point takes the + * loudest bin from halfway to the point before to halfway to the next, or – where the bins lie + * further apart than the points, down in the bass – the level between the two nearest. + */ +export function spectrumYs( + dbs: ArrayLike, + binHz: number, + xs: readonly number[], + area: EqArea, +): number[] { + const last = dbs.length - 1; + const level = (bin: number) => { + const db = dbs[Math.max(0, Math.min(last, bin))]!; + return Number.isFinite(db) ? db : SPECTRUM_FLOOR_DB; + }; + return xs.map((x, i) => { + const from = Math.ceil(xToFrequency(i > 0 ? (xs[i - 1]! + x) / 2 : x, area) / binHz); + const to = Math.floor(xToFrequency(i < xs.length - 1 ? (x + xs[i + 1]!) / 2 : x, area) / binHz); + let db = -Infinity; + for (let bin = from; bin <= Math.min(to, last); bin++) db = Math.max(db, level(bin)); + if (db === -Infinity) { + const at = xToFrequency(x, area) / binHz; + const below = Math.floor(at); + db = level(below) + (level(below + 1) - level(below)) * (at - below); + } + return spectrumY(db, area); + }); +} + +/** The frequency of the loudest bin, and how loud it is; null for a spectrum that is all silence. */ +export function spectrumPeak(dbs: ArrayLike, binHz: number): { hz: number; db: number } | null { + let best = -1; + for (let bin = 1; bin < dbs.length; bin++) + if (Number.isFinite(dbs[bin]) && (best < 0 || dbs[bin]! > dbs[best]!)) best = bin; + return best < 0 ? null : { hz: best * binHz, db: dbs[best]! }; +} diff --git a/src/viewport/SoundLibrary.ts b/src/viewport/SoundLibrary.ts index 30b4b235..ca4f4f7d 100644 --- a/src/viewport/SoundLibrary.ts +++ b/src/viewport/SoundLibrary.ts @@ -49,6 +49,7 @@ import { releaseEnvelope, stopProcessor, } from './fxNodes'; +import { type Spectrum, SpectrumTaps } from './spectrum'; import { wake } from './wake'; /** A voice is no louder than a sound item of a song turned all the way up. */ @@ -305,6 +306,8 @@ export class SoundLibrary implements AudioSink { private readonly ownerStops = new Map(); /** The tracks of the songs played so far (#334), by name. */ private readonly buses = new Map(); + /** The taps of the voices and buses playing now, and the node listened to (#374). */ + private readonly spectra = new SpectrumTaps(); private readonly peaked = new Map(); /** * The patterns and songs a game plays, each on a scheduler of its own. What it asks after are its @@ -531,6 +534,27 @@ export class SoundLibrary implements AudioSink { for (const bus of this.buses.values()) this.dropMeter(bus); } + /** + * Listens to the sound in front of the node `id` (`tapId`) and behind it (#374): a Parametric + * EQ's tab while it is seen. `spectrum` reads it, `unlisten` lets it go. + */ + listen(id: string): void { + this.spectra.listen(id); + } + + unlisten(): void { + this.spectra.stop(); + } + + spectrum(): Spectrum | null { + return this.spectra.read(); + } + + /** The node listened to, the taps playing, and how many of them feed an analyser (test hook). */ + spectrumTaps(): { listened: string; held: number; feeding: number } { + return this.spectra.info(); + } + /** How many buses are tapped for a meter (test hook). */ metered(): number { return [...this.buses.values()].filter((bus) => bus.meter).length; @@ -865,6 +889,7 @@ export class SoundLibrary implements AudioSink { tail = stageMade.output; knobs.push(...stageMade.knobs); if (stageMade.stop) made.stops.push(stageMade.stop); + if (stage.kind === 'tap') made.stops.push(this.spectra.hold(stage, stageMade.output)); nodes.push(stageMade.input, stageMade.output); } return { out: tail, latency }; @@ -1763,8 +1788,11 @@ export class SoundLibrary implements AudioSink { tail = made.output; knobs.push(...made.knobs); if (made.stop) wiring.drivers.push({ start: () => {}, stop: made.stop }); + if (stage.kind === 'tap') + wiring.drivers.push({ start: () => {}, stop: this.spectra.hold(stage, made.output) }); } - wiring.stages += chain.stages.length; + // A tap does nothing to the sound: the stages counted are those that do. + wiring.stages += chain.stages.filter((stage) => stage.kind !== 'tap').length; return tail; } diff --git a/src/viewport/fxNodes.ts b/src/viewport/fxNodes.ts index 8ba3e5a5..0d370222 100644 --- a/src/viewport/fxNodes.ts +++ b/src/viewport/fxNodes.ts @@ -112,6 +112,10 @@ export function stageNodes( follow?.({ envelope: stage.envelope, from: 0, to: 1 }, gain.gain, false); return { input: gain, output: gain, knobs: [] }; } + if (stage.kind === 'tap') { + const gain = gainNode(context, 1); + return { input: gain, output: gain, knobs: [] }; + } if (stage.kind === 'pass') return passNodes(context, stage, follow); if (stage.kind === 'delay') return delayNodes(context, stage); if (stage.kind === 'crush') return crushNodes(context, stage, follow); @@ -739,5 +743,5 @@ function stageLevels(stage: FxStage, into: number[]): void { into.push(stage.decay, stage.damping, stage.predelay, stage.mix, 1 - stage.mix); else if (stage.kind === 'saturate') into.push(driveGain(stage.drive), stage.tone, stage.level, stage.mix, 1 - stage.mix); - else if (stage.kind !== 'envelope') into.push(stage.gain); + else if (stage.kind !== 'envelope' && stage.kind !== 'tap') into.push(stage.gain); } diff --git a/src/viewport/spectrum.ts b/src/viewport/spectrum.ts new file mode 100644 index 00000000..403f5d77 --- /dev/null +++ b/src/viewport/spectrum.ts @@ -0,0 +1,121 @@ +/** + * The sound in front of a node and behind it, as an analyser hears it (#374). Every voice and bus + * passing a tap hands its node in for as long as it plays; while the node is listened to, they all + * feed one analyser for each side. Nothing is made before a tap of that node plays. + */ +import type { FxTap } from '../core/soundfx/nodes'; + +/** Points of one analysis: about 6 Hz apart at 48 kHz, close enough for a bass. */ +export const SPECTRUM_SIZE = 8192; + +type Side = FxTap['at']; +const SIDES: readonly Side[] = ['in', 'out']; + +interface Analysis { + sum: GainNode; + analyser: AnalyserNode; + data: Float32Array; +} + +/** Both sides of the node in dB, one level every `binHz`. */ +export interface Spectrum { + in: Float32Array; + out: Float32Array; + binHz: number; +} + +const keyOf = (id: string, at: Side) => `${at}:${id}`; + +/** Cuts one link; it may be gone with the rest of a voice already. */ +function cut(from: AudioNode, to: AudioNode): void { + try { + from.disconnect(to); + } catch { + // Not linked any more. + } +} + +export class SpectrumTaps { + /** The nodes of the taps that play now, by node and side. */ + private readonly held = new Map>(); + private listened = ''; + /** Made once a tap of the node listened to plays; kept while it is listened to. */ + private sides: Record | null = null; + + /** A tap of a voice or a bus as it starts to play; what it hands back is called once it stops. */ + hold(tap: FxTap, node: AudioNode): () => void { + const key = keyOf(tap.id, tap.at); + const nodes = this.held.get(key) ?? new Set(); + this.held.set(key, nodes); + nodes.add(node); + if (tap.id === this.listened) node.connect(this.analysis(node.context)[tap.at].sum); + return () => { + if (!nodes.delete(node)) return; + if (nodes.size === 0 && this.held.get(key) === nodes) this.held.delete(key); + if (tap.id === this.listened && this.sides) cut(node, this.sides[tap.at].sum); + }; + } + + /** Listens to the node `id` (`tapId`) and to no other. */ + listen(id: string): void { + if (id === this.listened) return; + this.stop(); + this.listened = id; + for (const at of SIDES) + for (const node of this.held.get(keyOf(id, at)) ?? []) + node.connect(this.analysis(node.context)[at].sum); + } + + /** Lets the analysers go: no tap feeds one any more. */ + stop(): void { + const { sides, listened } = this; + this.listened = ''; + this.sides = null; + if (!sides) return; + for (const at of SIDES) { + for (const node of this.held.get(keyOf(listened, at)) ?? []) cut(node, sides[at].sum); + sides[at].sum.disconnect(); + } + } + + /** + * The spectrum on both sides of the node listened to, smoothed by the analyser: once its sound has + * gone it falls away. Null before anything has played through it. + */ + read(): Spectrum | null { + const sides = this.sides; + if (!sides) return null; + for (const at of SIDES) sides[at].analyser.getFloatFrequencyData(sides[at].data); + return { + in: sides.in.data, + out: sides.out.data, + binHz: sides.in.analyser.context.sampleRate / SPECTRUM_SIZE, + }; + } + + /** The node listened to, the taps playing and how many of them feed an analyser (test hook). */ + info(): { listened: string; held: number; feeding: number } { + let held = 0; + for (const nodes of this.held.values()) held += nodes.size; + const feeding = this.sides + ? SIDES.reduce((sum, at) => sum + (this.held.get(keyOf(this.listened, at))?.size ?? 0), 0) + : 0; + return { listened: this.listened, held, feeding }; + } + + private analysis(context: BaseAudioContext): Record { + return (this.sides ??= { in: analysisOf(context), out: analysisOf(context) }); + } +} + +/** + * A sum the taps of one side go into, and the analyser behind it. With nothing coming in the sum + * still hands the analyser silence, so the picture falls away instead of standing still. + */ +function analysisOf(context: BaseAudioContext): Analysis { + const sum = context.createGain(); + const analyser = context.createAnalyser(); + analyser.fftSize = SPECTRUM_SIZE; + sum.connect(analyser); + return { sum, analyser, data: new Float32Array(analyser.frequencyBinCount) }; +} diff --git a/tests/core/instrument.test.ts b/tests/core/instrument.test.ts index 289a8c3a..06ee0bab 100644 --- a/tests/core/instrument.test.ts +++ b/tests/core/instrument.test.ts @@ -23,7 +23,7 @@ import { heldNote, noteFrequency, noteName } from '../../src/core/instrument/not import type { BehaviorAsset, InstrumentAsset, SoundFxAsset } from '../../src/core/model/Asset'; import { isPlaceableType } from '../../src/core/model/AssetTypes'; import { graphLookup } from '../../src/core/model/graphLookup'; -import { fxChainOf, type FxPlan } from '../../src/core/soundfx/nodes'; +import { fxChainOf, type FxPlan, tapId } from '../../src/core/soundfx/nodes'; import { paletteEntries } from '../../src/editor/graph/paletteEntries'; import { keyboardLayout, noteAt, trackerNote } from '../../src/editor/instrument/keys'; import type { AudioSink } from '../../src/runtime/systems/Audio'; @@ -515,6 +515,24 @@ describe('ports of an instrument', () => { f.edit.connect(other, 'out', gain, 'in'); expect(f.instrument.graph.edges.filter((e) => e.to.node === gain.id)).toHaveLength(1); }); + + it('taps a Parametric EQ in front and behind, named by the instrument and the node (#374)', () => { + const f = makeInstrument(); + const sample = nodeOf(f, 'InstrumentSample'); + sample.params.sound = f.sound('Piano', KICK).id; + const eq = f.edit.addNode('SoundFxParametricEq', { bands: [{ type: 'peaking', gain: 6 }] }); + f.edit.connect(sample, 'out', eq, 'in'); + f.edit.connect(eq, 'out', nodeOf(f, 'InstrumentOutput'), 'in'); + const id = tapId(f.instrument.id, eq.id); + const stages = way( + instrumentBuildFor(f.instrument, 60, instrumentSourcesOf(f.project)).layers[0], + )?.stages; + expect(stages?.map((s) => (s.kind === 'tap' ? `${s.at}:${s.id}` : s.kind))).toEqual([ + `in:${id}`, + 'pass', + `out:${id}`, + ]); + }); }); describe('Play note', () => { diff --git a/tests/core/parametric-eq.test.ts b/tests/core/parametric-eq.test.ts index b3d47ee0..c7459e09 100644 --- a/tests/core/parametric-eq.test.ts +++ b/tests/core/parametric-eq.test.ts @@ -1,21 +1,24 @@ /** - * The Parametric EQ (#367): up to ten bands on one node, compiled to filters in a row, and the curve - * the editor draws from them. + * The Parametric EQ (#367): up to ten bands on one node, compiled to filters in a row, the curve the + * editor draws from them, the curve in the node (#375), and the taps the spectrum is heard at (#374). */ import { describe, expect, it } from 'vitest'; import { CreateAssetCommand } from '../../src/core/commands/assetCommands'; import type { BehaviorAsset, SoundFxAsset } from '../../src/core/model/Asset'; import { compileSoundFx } from '../../src/core/soundfx/compile'; -import { fxChainOf } from '../../src/core/soundfx/nodes'; +import { fxChainOf, SoundFxParametricEq, tapId } from '../../src/core/soundfx/nodes'; import { + dbToY, type EqBand, eqBandDb, eqBandsOf, eqCurves, + fillShapes, + frequencyToX, MAX_EQ_BANDS, withBandSet, } from '../../src/core/soundfx/parametricEq'; -import { fillShapes } from '../../src/editor/soundfx/eqView'; +import { SPECTRUM_FLOOR_DB, spectrumPeak, spectrumYs } from '../../src/editor/soundfx/eqView'; import { graphEditor, makeProject, roundTrip } from '../helpers'; const band = (change: Partial): EqBand => ({ @@ -39,7 +42,7 @@ function eqLine(bands: unknown) { const output = edit.addNode('SoundFxOutput'); edit.connect(input, 'out', eq, 'in'); edit.connect(eq, 'out', output, 'in'); - return { ...f, fx }; + return { ...f, fx, eq }; } describe('the bands of a Parametric EQ', () => { @@ -65,15 +68,31 @@ describe('the bands of a Parametric EQ', () => { band({ type: 'highpass', frequency: 80, q: Math.SQRT1_2, slope: 24 }), band({ type: 'lowshelf', frequency: 200, gain: 0 }), ]); - const build = compileSoundFx(f.fx.graph); + const build = compileSoundFx(f.fx.graph, undefined, f.fx.id); + const id = tapId(f.fx.id, f.eq.id); expect(build.effects).toBe(1); expect(fxChainOf(build.compiled!).stages).toEqual([ + { kind: 'tap', id, at: 'in' }, { kind: 'pass', filter: 'peaking', stages: 1, cutoff: 1000, q: 1, gain: 6 }, { kind: 'pass', filter: 'highpass', stages: 2, cutoff: 80, q: 20 * Math.log10(Math.SQRT1_2), gain: 0 }, + { kind: 'tap', id, at: 'out' }, ]); expect(roundTrip(f.project)).toEqual(f.project); }); + it('are tapped in front of the node and behind it, behind its Wet and when bypassed too', () => { + const f = eqLine([band({ gain: 6 })]); + const id = tapId(f.fx.id, f.eq.id); + const kinds = () => + compileSoundFx(f.fx.graph, undefined, f.fx.id).compiled!.stages.map((s) => + s.kind === 'tap' ? `${s.at}:${s.id}` : s.kind, + ); + f.eq.params.wet = 0.5; + expect(kinds()).toEqual([`in:${id}`, 'wet', `out:${id}`]); + f.eq.params.bypass = true; + expect(kinds()).toEqual([`in:${id}`, `out:${id}`]); + }); + it('draw the curve Web Audio plays', () => { const peak = band({ gain: 6 }); expect(eqBandDb(peak, 1000)).toBeCloseTo(6, 3); @@ -112,3 +131,70 @@ describe('the bands of a Parametric EQ', () => { expect(fillShapes([0, 10, 20], [40, 30, 40], 50)).toEqual([[0, 50, 0, 40, 10, 30, 20, 40, 20, 50]]); }); }); + +describe('the curve in the node (#375)', () => { + const picture = SoundFxParametricEq.picture!; + const W = 164; + const area = { w: W, h: picture.height }; + const lineOf = (bands: unknown) => picture.shapes({ bands } as never, W).find((s) => s.kind === 'line')!; + + it('has its highest point where the peak is, as high as its gain', () => { + const line = lineOf([band({ gain: 6 })]); + let top = 1; + for (let i = 3; i < line.points.length; i += 2) if (line.points[i]! < line.points[top]!) top = i; + expect(Math.abs(line.points[top - 1]! - frequencyToX(1000, area))).toBeLessThanOrEqual(W / 96); + expect(line.points[top]).toBeCloseTo(dbToY(6, area), 1); + expect(line.points.length).toBe(97 * 2); + }); + + it('keeps its box without a band, flat on the 0 dB line', () => { + const shapes = picture.shapes({}, W); + const zero = dbToY(0, area); + expect(picture.height).toBe(60); + expect(shapes[0]).toEqual({ kind: 'guide', points: [0, zero, W, zero] }); + const line = shapes.find((s) => s.kind === 'line')!; + expect(line.points.filter((_, i) => i % 2 === 1).every((y) => y === zero)).toBe(true); + }); + + it('stays inside its box however deep a cut goes', () => { + const line = lineOf([band({ type: 'highpass', frequency: 2000, slope: 48 })]); + const ys = line.points.filter((_, i) => i % 2 === 1); + expect(Math.max(...ys)).toBe(picture.height); + expect(Math.min(...ys)).toBeGreaterThanOrEqual(0); + }); +}); + +describe('the spectrum behind the curve (#374)', () => { + const BIN = 48000 / 8192; + const area = { w: 600, h: 300 }; + const xs = Array.from({ length: 301 }, (_, i) => i * 2); + /** All silence but one loud bin at about 1 kHz. */ + const sine = () => { + const dbs = new Float32Array(4096).fill(-Infinity); + dbs[Math.round(1000 / BIN)] = -12; + return dbs; + }; + + it('finds the loudest bin, and nothing in silence', () => { + expect(spectrumPeak(sine(), BIN)!.hz).toBeCloseTo(Math.round(1000 / BIN) * BIN, 6); + expect(spectrumPeak(sine(), BIN)!.db).toBe(-12); + expect(spectrumPeak(new Float32Array(4096).fill(-Infinity), BIN)).toBeNull(); + }); + + it('draws the loud bin where 1 kHz is, at its level, and silence on the floor', () => { + const ys = spectrumYs(sine(), BIN, xs, area); + const top = ys.indexOf(Math.min(...ys)); + expect(Math.abs(xs[top]! - frequencyToX(1000, area))).toBeLessThanOrEqual(2); + expect(ys[top]).toBeCloseTo((-12 / SPECTRUM_FLOOR_DB) * area.h, 6); + expect(ys.filter((y) => y === area.h).length).toBeGreaterThan(xs.length - 4); + }); + + it('down in the bass, where bins lie further apart than the points, goes between them', () => { + const dbs = new Float32Array(4096).fill(-60); + dbs[4] = -20; + const at = (hz: number) => spectrumYs(dbs, BIN, [frequencyToX(hz, area)], area)[0]!; + // Bin 4 is 23.4 Hz, bin 5 29.3 Hz: halfway the level is halfway. + expect(at(4 * BIN)).toBeCloseTo((-20 / SPECTRUM_FLOOR_DB) * area.h, 6); + expect(at(4.5 * BIN)).toBeCloseTo((-40 / SPECTRUM_FLOOR_DB) * area.h, 6); + }); +}); diff --git a/tests/core/soundfx.test.ts b/tests/core/soundfx.test.ts index 87be99a3..636600f2 100644 --- a/tests/core/soundfx.test.ts +++ b/tests/core/soundfx.test.ts @@ -638,10 +638,12 @@ describe('Bypass and Wet (#354)', () => { for (const type of Object.keys(EFFECTS)) { const on = { gain: -6, semitones: 5, low: 3, pan: 1, depth: 50, bands: [{ gain: 6 }] }; expect([type, stagesOf(type, on).length > 0 || type === 'SoundFxPitch']).toEqual([type, true]); - expect([ + // The taps a Parametric EQ keeps do nothing to the sound (#374). + const bypassed = compileSoundFx(line([{ type, params: { ...on, bypass: true } }]).fx.graph).compiled!; + expect([type, { ...bypassed, stages: bypassed.stages.filter((s) => s.kind !== 'tap') }]).toEqual([ type, - compileSoundFx(line([{ type, params: { ...on, bypass: true } }]).fx.graph).compiled, - ]).toEqual([type, EMPTY_PLAN]); + EMPTY_PLAN, + ]); const def = SOUND_FX_NODES.find((d) => d.type === type)!; expect([def.dimmed?.({ bypass: true }), def.dimmed?.({})]).toEqual([true, false]); expect([def.hiddenParams?.({ bypass: true }), def.hiddenParams?.({})]).toEqual([[], ['bypass']]); diff --git a/tests/viewport/spectrum.test.ts b/tests/viewport/spectrum.test.ts new file mode 100644 index 00000000..6accac39 --- /dev/null +++ b/tests/viewport/spectrum.test.ts @@ -0,0 +1,92 @@ +/** + * The taps of the spectrum (#374): a voice's tap feeds the analyser of its side only while its node + * is listened to, and once that stops – or the voice does – nothing feeds one any more. + */ +import { describe, expect, it } from 'vitest'; +import type { FxTap } from '../../src/core/soundfx/nodes'; +import { SpectrumTaps } from '../../src/viewport/spectrum'; + +/** A node that remembers what it is linked to. */ +class FakeNode { + readonly links = new Set(); + fftSize = 0; + frequencyBinCount = 4; + constructor(readonly context: FakeContext) {} + connect(to: FakeNode): FakeNode { + this.links.add(to); + return to; + } + disconnect(to?: FakeNode): void { + if (!to) this.links.clear(); + else if (!this.links.delete(to)) throw new Error('not linked'); + } + getFloatFrequencyData(data: Float32Array): void { + data.fill(this.context.made.length); + } +} + +class FakeContext { + readonly sampleRate = 48000; + readonly made: FakeNode[] = []; + private make(): FakeNode { + const node = new FakeNode(this); + this.made.push(node); + return node; + } + createGain(): FakeNode { + return this.make(); + } + createAnalyser(): FakeNode { + return this.make(); + } +} + +const tap = (id: string, at: FxTap['at']): FxTap => ({ kind: 'tap', id, at }); + +describe('the taps of the spectrum', () => { + it('feed nothing and make nothing before their node is listened to', () => { + const context = new FakeContext(); + const taps = new SpectrumTaps(); + taps.hold(tap('fx:eq', 'in'), new FakeNode(context) as never); + expect(taps.info()).toEqual({ listened: '', held: 1, feeding: 0 }); + expect(context.made).toHaveLength(0); + expect(taps.read()).toBeNull(); + }); + + it('feed the analyser of their side while listened to, a voice started before as well', () => { + const context = new FakeContext(); + const taps = new SpectrumTaps(); + const before = new FakeNode(context); + taps.hold(tap('fx:eq', 'in'), before as never); + taps.listen('fx:eq'); + const after = new FakeNode(context); + taps.hold(tap('fx:eq', 'out'), after as never); + const other = new FakeNode(context); + taps.hold(tap('fx:other', 'out'), other as never); + expect(taps.info()).toEqual({ listened: 'fx:eq', held: 3, feeding: 2 }); + // Two sums, each into its analyser: the in side made first. + const [sumIn, , sumOut] = context.made; + expect([...before.links]).toEqual([sumIn]); + expect([...after.links]).toEqual([sumOut]); + expect(other.links.size).toBe(0); + expect(taps.read()?.binHz).toBeCloseTo(48000 / 8192, 9); + }); + + it('let go of the analysers once nobody listens, or once the voice stops', () => { + const context = new FakeContext(); + const taps = new SpectrumTaps(); + taps.listen('fx:eq'); + const voice = new FakeNode(context); + const stopVoice = taps.hold(tap('fx:eq', 'in'), voice as never); + const bus = new FakeNode(context); + taps.hold(tap('fx:eq', 'out'), bus as never); + stopVoice(); + stopVoice(); + expect(voice.links.size).toBe(0); + expect(taps.info()).toEqual({ listened: 'fx:eq', held: 1, feeding: 1 }); + taps.stop(); + expect(bus.links.size).toBe(0); + expect(taps.info()).toEqual({ listened: '', held: 1, feeding: 0 }); + expect(taps.read()).toBeNull(); + }); +}); diff --git a/tools/e2e/34-sound.mjs b/tools/e2e/34-sound.mjs index 864d5712..9d53c34b 100644 --- a/tools/e2e/34-sound.mjs +++ b/tools/e2e/34-sound.mjs @@ -572,7 +572,56 @@ export async function sound(t) { JSON.stringify({ eqDragged, eqUndone, eqWheeled }), ); await shot('34-parametric-eq'); + // #374: floated beside the Sound FX tab, the EQ's tab draws the spectrum of what passes the node: + // the blip's 880 Hz is the loudest bin in front of the EQ and behind it. Once the tab is not + // seen, no tap feeds an analyser any more, though the voice still plays. + await evaluate(`__gm.floatDocument(${JSON.stringify(`eq:${fxId}:fx-eq`)})`); + await evaluate('__gm.soundfx.setParam("depth", 0)'); + await evaluate('__gm.sound.stopAll()'); + const framesBefore = (await json('__gm.eq.spectrum()')).frames; + await evaluate('__gm.soundfx.play()'); + const spectrum = await waitFor( + () => json('__gm.eq.spectrum()'), + (s) => s.frames >= framesBefore + 3, + ); + await shot('34-eq-spectrum'); + const nearBlip = (hz) => Math.abs(hz - 880) <= spectrum.binHz; + check( + 'floated beside the Sound FX tab, the EQ draws the blip’s 880 Hz as the loudest bin on both sides', + spectrum.taps.listened === `${fxId}:fx-eq` && + spectrum.frames >= framesBefore + 3 && + nearBlip(spectrum.peakIn) && + nearBlip(spectrum.peakOut) && + spectrum.levelOut > -60, + JSON.stringify(spectrum), + ); + const eqWindow = (await json('__gm.floatWindows()')).find((w) => w.kind === 'eq'); + await click(eqWindow.dock.x, eqWindow.dock.y); + const unseen = await evaluate(`(async () => { + const frame = () => new Promise((done) => requestAnimationFrame(() => done())); + __gm.sound.stopAll(); + const before = __gm.sound.stats().started; + __gm.soundfx.play(); + for (let i = 0; i < 120 && __gm.sound.stats().started === before; i++) await frame(); + await frame(); + const seen = __gm.eq.spectrum().taps; + __gm.openDocument(${JSON.stringify(`asset:${fxId}`)}); + await frame(); + await frame(); + return { seen, unseen: __gm.eq.spectrum().taps, playing: __gm.sound.stats().playing }; + })()`); + check( + 'once the EQ’s tab is not seen, no tap feeds an analyser, though the voice still plays', + unseen.seen.held >= 2 && + unseen.seen.feeding === unseen.seen.held && + unseen.unseen.listened === '' && + unseen.unseen.held >= 2 && + unseen.unseen.feeding === 0 && + unseen.playing === 1, + JSON.stringify(unseen), + ); await evaluate(`__gm.openDocument(${JSON.stringify(`asset:${fxId}`)})`); + await shot('34-eq-node'); await evaluate('__gm.sound.stopAll()'); const eqBefore = await stats(); await click(fxPlayAt.x, fxPlayAt.y);