diff --git a/src/core/graph/nodes/action/PlaySound.ts b/src/core/graph/nodes/action/PlaySound.ts index faf48242..15d17815 100644 --- a/src/core/graph/nodes/action/PlaySound.ts +++ b/src/core/graph/nodes/action/PlaySound.ts @@ -63,7 +63,7 @@ export const PlaySound: NodeDef = { 'number of this one play, for Set sound FX parameters to change it while it plays; 0 where ' + 'nothing was played. "length" is how many seconds one run of the sound lasts, to hand to the ' + '"time" of Set sound FX parameters so a change spans the sound; 0 where nothing was played or ' + - 'the sound has not been loaded yet. A pitch changed while it plays makes it longer or shorter.', + 'the sound has not been loaded yet. A pitch never changes it.', inputs: [ exec(), soundPort(), diff --git a/src/core/graph/nodes/action/SetSoundFxParameters.ts b/src/core/graph/nodes/action/SetSoundFxParameters.ts index 19b5b48f..7593ac2c 100644 --- a/src/core/graph/nodes/action/SetSoundFxParameters.ts +++ b/src/core/graph/nodes/action/SetSoundFxParameters.ts @@ -19,7 +19,7 @@ export const SetSoundFxParameters: NodeDef = { 'Changes the sound FX parameters of a sound while it plays: the voice a Play sound handed out ' + 'glides to the values from a Sound FX parameters node. Keep the voice in a variable to reach it ' + 'from another event. A parameter not given keeps its value, a voice that has ended is not ' + - 'reached. A pitch changes by speed here, so the sound gets shorter or longer while it is moved. ' + + 'reached. A pitch keeps the length of the sound here too, but is heard about 20 ms after it is set. ' + 'time: seconds the values take to get there, along the chosen easing and on the clock of the ' + 'sound itself, so a sweep sounds the same every time; 0 is a quick glide.', inputs: [ diff --git a/src/core/soundfx/nodes.ts b/src/core/soundfx/nodes.ts index 12e07678..20550d92 100644 --- a/src/core/soundfx/nodes.ts +++ b/src/core/soundfx/nodes.ts @@ -15,14 +15,16 @@ export type FxStage = /** * A sound on its way through the graph: how many semitones it is played up or down, and what it - * passes, in order. `gain` is a factor, a filter's `gain` is in dB as Web Audio takes it. Behind a - * Mix (#316) it is two sounds, `b` making up the share `mix` of what goes on through `stages`. + * passes, in order. `gain` is a factor, a filter's `gain` is in dB as Web Audio takes it. `shifted` + * says the way has a Pitch at all – at 0 too, when that comes from a parameter – and so is played + * by the pitch worklet (#317). Behind a Mix (#316) it is two sounds, `b` making up the share `mix` + * of what goes on through `stages`. */ export type FxChain = - | { pitch: number; stages: readonly FxStage[] } + | { pitch: number; shifted: boolean; stages: readonly FxStage[] } | { mix: number; a: FxChain; b: FxChain; stages: readonly FxStage[] }; -export const DRY: FxChain = { pitch: 0, stages: [] }; +export const DRY: FxChain = { pitch: 0, shifted: false, stages: [] }; /** * A number of the graph as the compiler knows it: typed in, or a parameter of the Input that a @@ -106,7 +108,7 @@ function withStages(plan: FxPlan, stages: readonly FxStagePlan[]): FxPlan { return stages.length === 0 ? plan : { ...plan, stages: [...plan.stages, ...stages] }; } -/** A pitch is worked out on the sound as it starts, so behind a Mix it goes to both sounds. */ +/** A pitch is made where the sound is read, so behind a Mix it goes to both sounds. */ function withPitch(plan: FxPlan, semitones: FxAmount): FxPlan { if ('mix' in plan) return { ...plan, a: withPitch(plan.a, semitones), b: withPitch(plan.b, semitones) }; return { ...plan, pitch: [...plan.pitch, semitones] }; @@ -167,7 +169,7 @@ export const SoundFxPitch: SoundFxNodeDef = { label: 'Pitch', category: 'sound', description: - 'Plays the sound higher or lower, in semitones: 12 is an octave up, 0.5 a quarter tone. The sound keeps its length. The pitch is worked out on the sound as it starts, so an effect before the Pitch acts as if it came after it.', + 'Plays the sound higher or lower, in semitones: 12 is an octave up, 0.5 a quarter tone. The sound keeps its length, also when the pitch changes while it plays. The pitch is made where the sound is read, so an effect before the Pitch acts as if it came after it.', inputs: [audioIn, numberIn(SEMITONES)], outputs: [audioOut], params: [], @@ -270,7 +272,7 @@ export function fxChainOf(plan: FxPlan, params: FxParams = {}): FxChain { const stages = stagesOf(plan, params); if (!('mix' in plan)) { const pitch = plan.pitch.reduce((sum, amount) => sum + amountOf(amount, params), 0); - return { pitch, stages }; + return { pitch, shifted: plan.pitch.length > 0, stages }; } return { mix: amountOf(plan.mix, params), diff --git a/src/core/soundfx/pitchPlayer.ts b/src/core/soundfx/pitchPlayer.ts new file mode 100644 index 00000000..4f34ed3e --- /dev/null +++ b/src/core/soundfx/pitchPlayer.ts @@ -0,0 +1,411 @@ +/** Frames overlap four times: each output sample is made of four of them. */ +const OVERLAP = 4; +/** What four periodic Hann windows squared add up to wherever they overlap. */ +const WINDOW_SUM = 1.5; +/** + * How much of the way to the sound's own phase a peak is pulled each frame on the way back to 0 + * semitones: slow enough that the pitch wavers by a fifth of a semitone at most, done in half a second. + */ +const PULL = 0.1; +/** How close, in radians, the peaks have to be to the sound's phases before it is played as it is. */ +const ALIGNED = 0.05; + +/** + * Plays a sound higher or lower at its own length, the pitch free to change sample by sample + * (#317). A phase vocoder: the sound is cut into overlapping frames, and each frame's spectrum is + * moved up or down by the pitch. It is moved peak by peak, each with the bins around it (Laroche and + * Dolson's phase locking): a peak's phase runs on at its new frequency, so a tone carries on from + * frame to frame without a jump, and the bins around it keep their phase to it, so it stays one tone + * rather than beating against itself. It reads the sound itself rather than being fed it, so it can + * look ahead as far as a frame reaches and starts with the very first sample, no latency. A change of + * pitch is heard about half a frame later, though: a frame takes the pitch at the sample it starts on. + * + * At exactly 0 semitones a frame is the sound as it is, windowed in and out again, which the overlap + * adds back up to the samples themselves – and costs next to nothing. + */ +export class PitchPlayer { + private readonly size: number; + private readonly hop: number; + private readonly bins: number; + private readonly window: Float32Array; + /** The window squared and weighed so four overlapping frames add up to the sound. */ + private readonly dry: Float32Array; + private readonly reading: Float64Array; + /** A frame in time, turned by half its size: what goes into the transform and comes out of it. */ + private readonly real: Float64Array; + private readonly fft: RealFft; + private readonly re: Float64Array; + private readonly im: Float64Array; + private readonly magnitude: Float64Array; + private readonly frequency: Float64Array; + private readonly peaks: Int32Array; + private readonly nextRun: Float64Array; + /** + * Per channel: the phase of each bin in the last frame read, and for each bin the phase the peak + * that was moved onto it had – where a peak landing there next picks up. + */ + private readonly lastPhase: Float64Array[]; + private readonly runPhase: Float64Array[]; + /** Per channel: the samples of the frames still being added up, from the start of the newest frame on. */ + private readonly sum: Float32Array[]; + /** How far into the finished first hop of `sum` the output has been handed out. */ + private given: number; + /** Where the newest frame starts, in samples of the output. */ + private frameAt: number; + /** Whether `lastPhase` holds the frame before: a frame at 0 semitones does not need its phases. */ + private phased = false; + private primed = false; + /** How many samples of the output the sound lasts; a looping one never ends. */ + private readonly end: number; + + /** + * `channels` are the samples of the sound, `step` how many of them pass for one sample of the + * output (the sound's rate over the output's) and `rate` the output's rate, which sets the frame. + */ + constructor( + private readonly channels: readonly Float32Array[], + private readonly step: number, + rate: number, + private readonly loop: boolean, + ) { + // About 43 ms: a frame holds a few periods of a low note. + this.size = 2 ** Math.round(Math.log2(rate * 0.043)); + this.hop = this.size / OVERLAP; + this.bins = this.size / 2 + 1; + this.window = new Float32Array(this.size); + for (let j = 0; j < this.size; j++) this.window[j] = 0.5 - 0.5 * Math.cos((2 * Math.PI * j) / this.size); + this.dry = this.window.map((w) => (w * w) / WINDOW_SUM); + this.reading = new Float64Array(this.size); + this.real = new Float64Array(this.size); + this.fft = new RealFft(this.size); + this.re = new Float64Array(this.bins); + this.im = new Float64Array(this.bins); + this.magnitude = new Float64Array(this.bins); + this.frequency = new Float64Array(this.bins); + this.peaks = new Int32Array(this.bins); + this.nextRun = new Float64Array(this.bins); + this.lastPhase = channels.map(() => new Float64Array(this.bins)); + this.runPhase = channels.map(() => new Float64Array(this.bins)); + this.sum = channels.map(() => new Float32Array(this.size)); + this.given = this.hop; + this.frameAt = -this.size; + const length = channels[0]?.length ?? 0; + this.end = loop ? Infinity : Math.ceil(length / step); + } + + /** + * Fills `out` – one array per channel, all as long – with the next samples. `semitones` holds the + * pitch for each of them, or one value for all; a frame takes the pitch at the sample it starts on. + * Says whether the sound is still going: a sound that does not loop ends with its last sample. + */ + render(out: readonly Float32Array[], semitones: ArrayLike): boolean { + const count = out[0]?.length ?? 0; + const length = this.channels[0]?.length ?? 0; + if (!this.primed) { + // The frames before the first sample: what it is made of, so the start is not faded in. + for (let k = 1; k < OVERLAP; k++) this.frame(semitones[0] ?? 0); + this.given = this.hop; + this.primed = true; + } + for (let i = 0; i < count; i++) { + const at = this.frameAt + this.given; + if (at >= this.end || length === 0) { + for (const channel of out) channel.fill(0, i); + return false; + } + if (this.given === this.hop) this.frame(semitones.length > 1 ? semitones[i]! : (semitones[0] ?? 0)); + for (let c = 0; c < out.length; c++) + out[c]![i] = this.sum[Math.min(c, this.sum.length - 1)]![this.given] ?? 0; + this.given += 1; + } + return this.frameAt + this.given < this.end; + } + + /** Adds the next frame, a hop on from the last, and hands out the hop it finishes. */ + private frame(semitones: number): void { + this.frameAt += this.hop; + this.given = 0; + const ratio = 2 ** (semitones / 12); + if (ratio !== 1 && !this.phased) { + // Coming from 0 semitones, the frequency of each bin is read against the frame before. + for (let c = 0; c < this.channels.length; c++) this.analyse(c, this.frameAt - this.hop, true); + } + let apart = 0; + for (let c = 0; c < this.channels.length; c++) { + const sum = this.sum[c]!; + sum.copyWithin(0, this.hop); + sum.fill(0, this.size - this.hop); + if (ratio === 1 && !this.phased) this.addDry(c, sum); + else apart = Math.max(apart, this.addMoved(c, sum, ratio)); + } + // Back at 0 semitones the frames go on being made until their phases have come round to the + // sound's: switching to the sound itself before that would cancel against the frames still sounding. + this.phased = ratio !== 1 || apart > ALIGNED; + } + + /** At 0 semitones a frame is the sound, windowed twice and weighed so the overlap adds up to it. */ + private addDry(c: number, sum: Float32Array): void { + const { size, dry } = this; + const frame = this.readFrame(c, this.frameAt); + for (let j = 0; j < size; j++) sum[j]! += frame[j]! * dry[j]!; + } + + /** + * A frame with its spectrum moved by `ratio`. At a ratio of 1 every peak is pulled a little towards + * the phase the sound has there; says how far the loudest peaks still are from it, in radians. + */ + private addMoved(c: number, sum: Float32Array, ratio: number): number { + const { size, bins, re, im, real, magnitude, frequency, window, peaks, nextRun } = this; + this.analyse(c, this.frameAt, false); + const phase = this.lastPhase[c]!; + const run = this.runPhase[c]!; + const advance = (2 * Math.PI * this.hop) / size; + // A peak is louder than the two bins on either side; the two at each end are never one. + let count = 0; + let loudest = 0; + for (let k = 2; k < bins - 2; k++) { + const m = magnitude[k]!; + if ( + m > magnitude[k - 1]! && + m >= magnitude[k + 1]! && + m > magnitude[k - 2]! && + m >= magnitude[k + 2]! + ) { + peaks[count++] = k; + if (m > loudest) loudest = m; + } + } + re.fill(0); + im.fill(0); + nextRun.set(run); + let apart = 0; + // Each peak owns the bins down to the lowest one between it and the next peak; moved together, + // two that land on one bin add up. + let start = 0; + for (let i = 0; i < count; i++) { + const peak = peaks[i]!; + let end = bins; + if (i + 1 < count) { + end = peak + 1; + for (let k = peak + 1; k < peaks[i + 1]!; k++) if (magnitude[k]! < magnitude[end]!) end = k; + } + const moved = frequency[peak]! * ratio; + const to = Math.round(moved); + if (to >= 0 && to < bins) { + let base = wrap(run[to]! + moved * advance); + if (ratio === 1) { + const off = wrap(phase[peak]! - base); + base = wrap(base + PULL * off); + if (magnitude[peak]! > loudest * 0.01) apart = Math.max(apart, Math.abs(off)); + } + const turn = base - phase[peak]!; + for (let k = Math.max(start, peak - to); k < Math.min(end, bins + peak - to); k++) { + const t = k + to - peak; + const turned = turn + phase[k]!; + re[t]! += magnitude[k]! * Math.cos(turned); + im[t]! += magnitude[k]! * Math.sin(turned); + nextRun[t] = base; + } + } + start = end; + } + run.set(nextRun); + this.fft.inverse(re, im, real); + const scale = 1 / (size * WINDOW_SUM); + const half = size / 2; + for (let j = 0; j < half; j++) { + sum[j]! += real[j + half]! * window[j]! * scale; + sum[j + half]! += real[j]! * window[j + half]! * scale; + } + return apart; + } + + /** + * The spectrum of the frame starting at `from`: each bin's magnitude, its phase and its true + * frequency in bins, read from how far its phase moved since the frame a hop before. `fresh` starts + * the running phases where the sound has them, so going from 0 semitones to another pitch slowly + * carries on without a jump. The frame is turned by half its size first, so a phase is the one at + * its middle: the bins of one tone then share it, wherever between two bins the tone lies. + */ + private analyse(c: number, from: number, fresh: boolean): void { + const { size, bins, re, im, real, window, magnitude, frequency } = this; + const half = size / 2; + const frame = this.readFrame(c, from); + for (let j = 0; j < half; j++) { + real[j + half] = frame[j]! * window[j]!; + real[j] = frame[j + half]! * window[j + half]!; + } + this.fft.forward(real, re, im); + const last = this.lastPhase[c]!; + const run = this.runPhase[c]!; + const advance = (2 * Math.PI * this.hop) / size; + for (let k = 0; k < bins; k++) { + const phase = Math.atan2(im[k]!, re[k]!); + const moved = wrap(phase - last[k]! - k * advance); + last[k] = phase; + if (fresh) run[k] = phase; + magnitude[k] = Math.sqrt(re[k]! * re[k]! + im[k]! * im[k]!); + frequency[k] = k + moved / advance; + } + } + + /** + * Channel `c` for the frame starting at output sample `from`, each sample read between two of the + * sound's; outside the sound it is silence, or the loop again. + */ + private readFrame(c: number, from: number): Float64Array { + const { reading, size, step, loop } = this; + const samples = this.channels[c]!; + const length = samples.length; + for (let j = 0; j < size; j++) { + let position = (from + j) * step; + if (position < 0 || position >= length) { + if (!loop) { + reading[j] = 0; + continue; + } + position -= Math.floor(position / length) * length; + } + const below = Math.floor(position); + const fraction = position - below; + if (fraction === 0) { + reading[j] = samples[below]!; + continue; + } + const next = below + 1 < length ? samples[below + 1]! : loop ? samples[0]! : 0; + reading[j] = samples[below]! * (1 - fraction) + next * fraction; + } + return reading; + } +} + +function wrap(phase: number): number { + return phase - 2 * Math.PI * Math.round(phase / (2 * Math.PI)); +} + +/** + * The FFT of a real frame, done as a complex one of half the size: the even samples go in as the + * real part, the odd ones as the imaginary, and the two halves are taken apart again afterwards. + * Only the bins up to the middle are handed out; the rest mirror them. + */ +class RealFft { + private readonly half: number; + private readonly fft: Fft; + private readonly zr: Float64Array; + private readonly zi: Float64Array; + private readonly cos: Float64Array; + private readonly sin: Float64Array; + + constructor(size: number) { + this.half = size / 2; + this.fft = new Fft(this.half); + this.zr = new Float64Array(this.half); + this.zi = new Float64Array(this.half); + this.cos = new Float64Array(this.half + 1); + this.sin = new Float64Array(this.half + 1); + for (let k = 0; k <= this.half; k++) { + this.cos[k] = Math.cos((2 * Math.PI * k) / size); + this.sin[k] = Math.sin((2 * Math.PI * k) / size); + } + } + + /** `re` and `im` get the bins 0 to half the size. */ + forward(input: Float64Array, re: Float64Array, im: Float64Array): void { + const { half, zr, zi, cos, sin } = this; + for (let n = 0; n < half; n++) { + zr[n] = input[2 * n]!; + zi[n] = input[2 * n + 1]!; + } + this.fft.transform(zr, zi, false); + for (let k = 0; k <= half; k++) { + const a = k % half; + const b = (half - k) % half; + // The even samples' spectrum and the odd ones', taken apart from the one transform. + const er = (zr[a]! + zr[b]!) / 2; + const ei = (zi[a]! - zi[b]!) / 2; + const or = (zi[a]! + zi[b]!) / 2; + const oi = -(zr[a]! - zr[b]!) / 2; + // Plus the odd ones turned by the bin's share of a full circle. + re[k] = er + or * cos[k]! + oi * sin[k]!; + im[k] = ei + oi * cos[k]! - or * sin[k]!; + } + } + + /** The frame back from the bins 0 to half the size, times the size as an unscaled inverse is. */ + inverse(re: Float64Array, im: Float64Array, output: Float64Array): void { + const { half, zr, zi, cos, sin } = this; + for (let k = 0; k < half; k++) { + const m = half - k; + const er = (re[k]! + re[m]!) / 2; + const ei = (im[k]! - im[m]!) / 2; + const dr = (re[k]! - re[m]!) / 2; + const di = (im[k]! + im[m]!) / 2; + const or = dr * cos[k]! - di * sin[k]!; + const oi = dr * sin[k]! + di * cos[k]!; + zr[k] = er - oi; + zi[k] = ei + or; + } + this.fft.transform(zr, zi, true); + for (let n = 0; n < half; n++) { + output[2 * n] = 2 * zr[n]!; + output[2 * n + 1] = 2 * zi[n]!; + } + } +} + +/** A radix-2 complex FFT of a fixed size, its turns and the order it reads in worked out once. */ +class Fft { + private readonly cos: Float64Array; + private readonly sin: Float64Array; + private readonly order: Uint32Array; + + constructor(private readonly size: number) { + this.cos = new Float64Array(size / 2); + this.sin = new Float64Array(size / 2); + for (let i = 0; i < size / 2; i++) { + this.cos[i] = Math.cos((2 * Math.PI * i) / size); + this.sin[i] = Math.sin((2 * Math.PI * i) / size); + } + this.order = new Uint32Array(size); + const bits = Math.log2(size); + for (let i = 0; i < size; i++) { + let reversed = 0; + for (let b = 0; b < bits; b++) reversed |= ((i >> b) & 1) << (bits - 1 - b); + this.order[i] = reversed; + } + } + + /** In place; the inverse is not divided by the size. */ + transform(re: Float64Array, im: Float64Array, inverse: boolean): void { + const { size, order } = this; + for (let i = 0; i < size; i++) { + const j = order[i]!; + if (j <= i) continue; + const r = re[i]!; + re[i] = re[j]!; + re[j] = r; + const m = im[i]!; + im[i] = im[j]!; + im[j] = m; + } + const sign = inverse ? 1 : -1; + for (let span = 2; span <= size; span *= 2) { + const half = span / 2; + const stride = size / span; + for (let start = 0; start < size; start += span) { + for (let k = 0; k < half; k++) { + const wr = this.cos[k * stride]!; + const wi = sign * this.sin[k * stride]!; + const a = start + k; + const b = a + half; + const tr = re[b]! * wr - im[b]! * wi; + const ti = re[b]! * wi + im[b]! * wr; + re[b] = re[a]! - tr; + im[b] = im[a]! - ti; + re[a] = re[a]! + tr; + im[a] = im[a]! + ti; + } + } + } + } +} diff --git a/src/core/soundfx/pitchShift.ts b/src/core/soundfx/pitchShift.ts deleted file mode 100644 index bb723e15..00000000 --- a/src/core/soundfx/pitchShift.ts +++ /dev/null @@ -1,95 +0,0 @@ -/** Half a frame, in seconds: the hop between frames. A frame is longer than the period of a low note. */ -const HOP_SECONDS = 0.02; -/** How far a frame may move to fit the one before it: a period of 80 Hz. */ -const REACH_SECONDS = 0.0125; -/** The first search looks at every fourth offset and sample, the second at every one around the best. */ -const COARSE = 4; - -/** - * One channel of a sound played `semitones` higher or lower at its own length (#312). The sound is - * first read faster or slower, which sets the pitch and gets the length wrong, and then stretched - * back by WSOLA: frames half a frame apart are laid over each other under a Hann window, each one - * moved a little to where it continues the frame before it, so a tone carries on without a jump in - * phase. A sound that loops reads on across its end into its start, so the seam stays where it was. - */ -export function pitchShift( - samples: Float32Array, - sampleRate: number, - semitones: number, - loop = false, -): Float32Array { - const length = samples.length; - const out = new Float32Array(length); - if (length === 0) return out; - const ratio = 2 ** (semitones / 12); - - const read = (index: number): number => { - if (loop) return samples[((index % length) + length) % length]!; - return index >= 0 && index < length ? samples[index]! : 0; - }; - const hop = Math.max(1, Math.round(HOP_SECONDS * sampleRate)); - const size = hop * 2; - const reach = Math.round(REACH_SECONDS * sampleRate); - // The pitched sound with room on either side for a frame that reaches past an end, so the loops - // below index it directly. - const pad = size + reach + COARSE; - const span = Math.max(1, Math.round(length / ratio)); - const pitched = new Float32Array(span + 2 * pad); - for (let i = -pad; i < span + pad; i++) { - if (!loop && (i < 0 || i >= span)) continue; - const position = (((i % span) + span) % span) * ratio; - const below = Math.floor(position); - const fraction = position - below; - pitched[i + pad] = read(below) * (1 - fraction) + read(below + 1) * fraction; - } - - const window = new Float32Array(size); - for (let j = 0; j < size; j++) window[j] = 0.5 - 0.5 * Math.cos((2 * Math.PI * j) / size); - const lowest = 0; - const highest = pitched.length - size; - const within = (from: number) => Math.max(lowest, Math.min(highest, from + pad)); - - let previous: number | null = null; - for (let start = -hop; start < length; start += hop) { - let from = within(Math.round((start + hop) / ratio) - hop); - if (previous !== null) from = bestFit(pitched, previous + hop, from, reach, hop, lowest, highest); - for (let j = Math.max(0, -start); j < size && start + j < length; j++) - out[start + j]! += window[j]! * pitched[from + j]!; - previous = from; - } - return out; -} - -/** - * Where near `around` the sound looks most like it does `hop` samples on from `follow`, searched - * coarsely and then finely, never outside `lowest`..`highest`. - */ -function bestFit( - sound: Float32Array, - follow: number, - around: number, - reach: number, - hop: number, - lowest: number, - highest: number, -): number { - const score = (from: number, stride: number): number => { - let sum = 0; - for (let j = 0; j < hop; j += stride) sum += sound[from + j]! * sound[follow + j]!; - return sum; - }; - const search = (centre: number, span: number, stride: number): number => { - let best = centre; - let bestScore = score(centre, stride); - for (let offset = -span; offset <= span; offset += stride) { - if (centre + offset < lowest || centre + offset > highest) continue; - const candidate = score(centre + offset, stride); - if (candidate > bestScore) { - best = centre + offset; - bestScore = candidate; - } - } - return best; - }; - return search(search(around, reach, COARSE), COARSE - 1, 1); -} diff --git a/src/editor/debugHook.ts b/src/editor/debugHook.ts index 0e6d5d8e..a70ec228 100644 --- a/src/editor/debugHook.ts +++ b/src/editor/debugHook.ts @@ -43,6 +43,7 @@ export function installDebugHook(app: Application, state: EditorState, editor: E duration: (file: string) => soundLibrary.duration(file), /** Where the knobs of each voice of that file are now, oldest first, a pitch as its speed (#315). */ levels: (file: string) => soundLibrary.levels(file), + pitchWorkletUrl: () => soundLibrary.pitchWorkletUrl(), /** Decodes ahead of a check, so a play can be asserted without racing the network. */ load: (file: string) => soundLibrary.load(file).then((buffer) => buffer?.duration ?? null), stopAll: () => soundLibrary.stopAll(), diff --git a/src/viewport/SoundLibrary.ts b/src/viewport/SoundLibrary.ts index 19344dfd..27551061 100644 --- a/src/viewport/SoundLibrary.ts +++ b/src/viewport/SoundLibrary.ts @@ -3,16 +3,16 @@ import { fxChainOf, type FxChain, type FxParams, + type FxPlan, type FxStage, type FxVoice, } from '../core/soundfx/nodes'; import type { EaseShare } from '../core/model/animation'; -import { pitchShift } from '../core/soundfx/pitchShift'; import type { AudioSink } from '../runtime/systems/Audio'; +import type { PitchOptions } from './pitchWorklet'; +import pitchWorklet from './pitchWorklet.ts?worker&url'; const DEFAULT_BASE = '/game-maker/data/sounds'; -/** How many pitched copies are kept at once. */ -const SHIFTED_KEPT = 32; /** Seconds a changed parameter takes to get most of the way there: quick, but no click (#315). */ const GLIDE = 0.03; /** The rate a sound loaded ahead of its first play is decoded at; a source resamples it on the way out. */ @@ -20,13 +20,22 @@ const DECODE_RATE = 48000; /** * One number of a voice that can still change while it plays (#315): a level, a share of a Mix, the - * gain of a band – or the pitch of a source, which glides by speed from the pitch its copy was made at. - * `even` says what a timed change runs evenly in: a share or a band in what it holds, a gain stage in - * decibels (its node holds the factor), a pitch in semitones (its node holds the speed). + * gain of a band, the pitch of a way in semitones. `even` says what a timed change runs evenly in: a + * gain stage in decibels (its node holds the factor), everything else in what it holds. A way whose + * pitch has no worklet to play it has no `param`: it keeps its own pitch. */ -type Knob = - | { kind: 'level'; param: AudioParam; target: number; even: 'value' | 'ratio' } - | { kind: 'pitch'; source: AudioBufferSourceNode; base: number; target: number }; +interface Knob { + param: AudioParam | null; + target: number; + even: 'value' | 'ratio'; +} + +/** One way's sound as a voice holds it: a buffer source, or the pitch worklet where the way has a Pitch (#317). */ +interface Source { + start(): void; + stop(): void; + ended(then: () => void): void; +} /** * One sound in flight: its sources, its own gain, the file it came from and who is making it. A @@ -39,7 +48,7 @@ interface Voice { file: string; /** The behavior this voice belongs to; `''` for one nothing owns (see `AudioSink.play`). */ owner: string; - sources: AudioBufferSourceNode[]; + sources: Source[]; gain: GainNode; fx: FxVoice; knobs: Knob[]; @@ -51,7 +60,8 @@ interface Wiring { file: string; buffer: AudioBuffer; loop: boolean; - sources: { source: AudioBufferSourceNode; pitch: number }[]; + sources: Source[]; + pitches: number[]; knobs: Knob[]; stages: number; } @@ -79,14 +89,20 @@ interface Pending { * has been interacted with, and an editor that was only ever looked at should not be holding an * audio device open. What `preload` fetches before that is decoded by an offline context, which * holds no device. + * + * A way with a Pitch is played by a worklet (#317), loaded into the context with the first play that + * needs it; that play waits for it as it would for its bytes. */ export class SoundLibrary implements AudioSink { private base = DEFAULT_BASE; private context: AudioContext | null = null; private offline: OfflineAudioContext | null = null; private readonly buffers = new Map(); - /** Pitched copies of decoded sounds, oldest use first (see `pitched`). */ - private readonly shifted = new Map(); + /** The pitch worklet being loaded, and whether it is there once it is known; null before that. */ + private pitching: Promise | null = null; + private canPitch: boolean | null = null; + /** The files whose samples the worklet holds already, so the next voice of one is sent without. */ + private readonly sent = new Set(); private readonly loading = new Map>(); /** Never retried, so one broken reference cannot spam the network. */ private readonly failed = new Set(); @@ -124,14 +140,15 @@ export class SoundLibrary implements AudioSink { if (file === '' || this.failed.has(file)) return 0; const id = this.nextId++; const buffer = this.buffers.get(file); - if (buffer) { + const pitching = pitches(fx.plan) ? this.pitchWorklet() : null; + if (buffer && (!pitching || this.canPitch !== null)) { this.start(id, file, buffer, volume, loop, owner, fx); return id; } const askedAt = this.cancelsFor(file, owner); const waiting: Pending = { id, fx: { plan: fx.plan, params: { ...fx.params } }, timed: [] }; this.pending.push(waiting); - void this.load(file).then((loaded) => { + void Promise.all([this.load(file), pitching]).then(([loaded]) => { this.pending = this.pending.filter((other) => other !== waiting); // A `Stop sound` – or the destruction of the behavior that asked – that happened while the // bytes were in flight has to win, or a sound the run already ended would start afterwards. @@ -197,6 +214,8 @@ export class SoundLibrary implements AudioSink { tuned: number; decoded: number; failed: number; + /** Whether the pitch worklet is there; null while nothing has asked for it or it is loading. */ + worklet: boolean | null; last: LastVoice | null; } { return { @@ -206,20 +225,24 @@ export class SoundLibrary implements AudioSink { tuned: this.tuned, decoded: this.buffers.size, failed: this.failed.size, + worklet: this.canPitch, last: this.last, }; } /** - * Where the knobs of each voice of that file are right now, oldest first, a pitch as the speed it - * plays at – what a check reads to see a change arrive at one voice and not another (test hook). + * Where the knobs of each voice of that file are right now, oldest first, a pitch in semitones – + * what a check reads to see a change arrive at one voice and not another (test hook). */ levels(file: string): number[][] { return this.voices .filter((voice) => voice.file === file) - .map((voice) => - voice.knobs.map((knob) => (knob.kind === 'pitch' ? knob.source.playbackRate : knob.param).value), - ); + .map((voice) => voice.knobs.map((knob) => knob.param?.value ?? knob.target)); + } + + /** Where the pitch worklet is served from, for a check that runs it in an offline context (test hook). */ + pitchWorkletUrl(): string { + return pitchWorklet; } /** How long a decoded sound runs, in seconds; null while it has not been loaded (test hook). */ @@ -288,23 +311,23 @@ export class SoundLibrary implements AudioSink { if (context.state === 'suspended') void context.resume(); const gain = context.createGain(); gain.gain.value = Math.max(0, Math.min(1, volume)); - const wiring: Wiring = { context, file, buffer, loop, sources: [], knobs: [], stages: 0 }; + const wiring: Wiring = { context, file, buffer, loop, sources: [], pitches: [], knobs: [], stages: 0 }; this.wire(wiring, fxChainOf(fx.plan, fx.params)).connect(gain).connect(context.destination); - const sources = wiring.sources.map((made) => made.source); + const { sources } = wiring; this.last = { file, volume: gain.gain.value, loop, - pitches: wiring.sources.map((made) => made.pitch), + pitches: wiring.pitches, stages: wiring.stages, - duration: sources[0]!.buffer!.duration, + duration: buffer.duration, }; const voice: Voice = { id, file, owner, sources, gain, fx, knobs: wiring.knobs }; let playing = sources.length; for (const source of sources) - source.onended = () => { + source.ended(() => { if (--playing === 0) this.voices = this.voices.filter((other) => other !== voice); - }; + }); this.voices.push(voice); this.started += 1; for (const source of sources) source.start(); @@ -312,7 +335,7 @@ export class SoundLibrary implements AudioSink { /** * The nodes a chain becomes, up to the one that carries its sound on. Every way has a source of - * its own, so each can glide to its own pitch; a Mix weighs each of its sounds by its share before + * its own, so each can have its own pitch; a Mix weighs each of its sounds by its share before * they are summed. */ private wire(wiring: Wiring, chain: FxChain): AudioNode { @@ -323,17 +346,16 @@ export class SoundLibrary implements AudioSink { const wayA = this.wire(wiring, chain.a); const shareA = gainNode(context, 1 - chain.mix); wayA.connect(shareA).connect(tail); - knobs.push({ kind: 'level', param: shareA.gain, target: 1 - chain.mix, even: 'value' }); + knobs.push({ param: shareA.gain, target: 1 - chain.mix, even: 'value' }); const wayB = this.wire(wiring, chain.b); const shareB = gainNode(context, chain.mix); wayB.connect(shareB).connect(tail); - knobs.push({ kind: 'level', param: shareB.gain, target: chain.mix, even: 'value' }); - } else tail = this.source(wiring, chain.pitch); + knobs.push({ param: shareB.gain, target: chain.mix, even: 'value' }); + } else tail = this.source(wiring, chain.pitch, chain.shifted); for (const stage of chain.stages) { const [node, param] = fxNode(context, stage); tail = tail.connect(node); knobs.push({ - kind: 'level', param, target: stage.gain, even: stage.kind === 'gain' ? 'ratio' : 'value', @@ -343,20 +365,60 @@ export class SoundLibrary implements AudioSink { return tail; } - private source(wiring: Wiring, pitch: number): AudioBufferSourceNode { + /** + * A way's sound: the pitch worklet where the way has a Pitch, which keeps the length whatever the + * pitch does (#317), and otherwise the buffer as it is. Without the worklet a pitched way plays at + * its own pitch rather than not at all. + */ + private source(wiring: Wiring, pitch: number, shifted: boolean): AudioNode { const { context, file, buffer, loop } = wiring; + wiring.pitches.push(pitch); + if (shifted && this.canPitch) { + const options: PitchOptions = { file, step: buffer.sampleRate / context.sampleRate, loop }; + if (!this.sent.has(file)) { + options.channels = Array.from({ length: buffer.numberOfChannels }, (_, c) => + buffer.getChannelData(c), + ); + this.sent.add(file); + } + const node = new AudioWorkletNode(context, 'pitch-player', { + numberOfInputs: 0, + outputChannelCount: [buffer.numberOfChannels], + parameterData: { pitch }, + processorOptions: options, + }); + wiring.sources.push({ + start: () => {}, + stop: () => { + node.port.postMessage('stop'); + node.disconnect(); + }, + ended: (then) => { + node.port.onmessage = (event: MessageEvent) => { + if (event.data === 'ended') then(); + }; + }, + }); + wiring.knobs.push({ param: node.parameters.get('pitch')!, target: pitch, even: 'value' }); + return node; + } const source = context.createBufferSource(); - source.buffer = pitch === 0 ? buffer : this.pitched(context, file, buffer, pitch, loop); + source.buffer = buffer; source.loop = loop; - wiring.sources.push({ source, pitch }); - wiring.knobs.push({ kind: 'pitch', source, base: pitch, target: pitch }); + wiring.sources.push({ + start: () => source.start(), + stop: () => source.stop(), + ended: (then) => { + source.onended = then; + }, + }); + if (shifted) wiring.knobs.push({ param: null, target: pitch, even: 'value' }); return source; } /** * Sends every knob of a voice towards what its values say now, in `time` seconds of the audio - * clock along `ease`, or else in a quick glide. A pitch cannot get a new copy in the middle of a sound, so it - * glides by speed: the voice gets that much shorter or longer. + * clock along `ease`, or else in a quick glide. */ private glide(voice: Voice, time: number, ease: EaseShare): void { const levels = levelsOf(fxChainOf(voice.fx.plan, voice.fx.params)); @@ -365,45 +427,27 @@ export class SoundLibrary implements AudioSink { const level = levels[i]; if (level === undefined || level === knob.target) return; knob.target = level; - if (knob.kind === 'pitch') - move(knob.source.playbackRate, 2 ** ((level - knob.base) / 12), now, time, true, ease); - else move(knob.param, level, now, time, knob.even === 'ratio', ease); + if (knob.param) move(knob.param, level, now, time, knob.even === 'ratio', ease); }); } - /** - * The sound at another pitch and its own length (#312), worked out once per file, pitch and loop - * and kept for the next voice. The oldest goes first, so a pitch drawn at random every play cannot - * fill the memory. - */ - private pitched( - context: AudioContext, - file: string, - buffer: AudioBuffer, - pitch: number, - loop: boolean, - ): AudioBuffer { - const key = `${file}\u0000${pitch}\u0000${loop}`; - const held = this.shifted.get(key); - if (held) { - this.shifted.delete(key); - this.shifted.set(key, held); - return held; - } - const shifted = context.createBuffer(buffer.numberOfChannels, buffer.length, buffer.sampleRate); - for (let channel = 0; channel < buffer.numberOfChannels; channel++) - shifted - .getChannelData(channel) - .set(pitchShift(buffer.getChannelData(channel), buffer.sampleRate, pitch, loop)); - this.shifted.set(key, shifted); - if (this.shifted.size > SHIFTED_KEPT) this.shifted.delete(this.shifted.keys().next().value!); - return shifted; - } - private audio(): AudioContext { return (this.context ??= new AudioContext()); } + /** Loads the pitch worklet into the context once; says whether it is there. */ + private pitchWorklet(): Promise { + return (this.pitching ??= (async () => { + try { + await this.audio().audioWorklet.addModule(pitchWorklet); + return (this.canPitch = true); + } catch (error: unknown) { + console.error('[sounds] cannot load the pitch worklet, a Pitch is not heard', error); + return (this.canPitch = false); + } + })()); + } + /** The context that is already playing, or else one that decodes without opening a device. */ private decoder(): BaseAudioContext { if (this.context || typeof OfflineAudioContext === 'undefined') return this.audio(); @@ -421,13 +465,18 @@ interface LastVoice { file: string; volume: number; loop: boolean; - /** Semitones of each source, one per way (#316), and how many gain and filter nodes it passed. */ + /** Semitones of each way (#316), and how many gain and filter nodes it passed. */ pitches: number[]; stages: number; - /** Seconds of what was played: a pitched voice is as long as the sound itself (#312). */ + /** Seconds of what was played: a pitched voice is as long as the sound itself (#312, #317). */ duration: number; } +/** Whether any way of a plan has a Pitch, and so needs the worklet. */ +function pitches(plan: FxPlan): boolean { + return 'mix' in plan ? pitches(plan.a) || pitches(plan.b) : plan.pitch.length > 0; +} + function gainNode(context: AudioContext, level: number): GainNode { const gain = context.createGain(); gain.gain.value = level; @@ -451,7 +500,7 @@ function fxNode(context: AudioContext, stage: FxStage): [AudioNode, AudioParam] /** * Sends one knob from where it is right now to `to`. Whatever was still planned for it is dropped * first, so a change never queues behind an older one. A timed change runs along `ease` – a `ratio` - * one in its logarithm, which is what decibels and semitones are. Linear and step are the clock's + * one in its logarithm, which is what decibels are. Linear and step are the clock's * own ramps, every other easing is handed over as a curve of samples. */ function move( @@ -505,7 +554,7 @@ function levelsOf(chain: FxChain, into: number[] = []): number[] { into.push(1 - chain.mix); levelsOf(chain.b, into); into.push(chain.mix); - } else into.push(chain.pitch); + } else if (chain.shifted) into.push(chain.pitch); for (const stage of chain.stages) into.push(stage.gain); return into; } diff --git a/src/viewport/pitchWorklet.ts b/src/viewport/pitchWorklet.ts new file mode 100644 index 00000000..3499953e --- /dev/null +++ b/src/viewport/pitchWorklet.ts @@ -0,0 +1,59 @@ +/** + * The audio thread's side of a pitched voice (#317): plays one sound through a `PitchPlayer`, its + * pitch the AudioParam `pitch` in semitones, so it glides, ramps and eases like any other knob. Loaded + * into the context once by the sound library; nothing on the page imports it. + */ +import { PitchPlayer } from '../core/soundfx/pitchPlayer'; + +/** What the audio thread has and the page's types do not. */ +declare const sampleRate: number; +declare class AudioWorkletProcessor { + readonly port: MessagePort; +} +declare function registerProcessor( + name: string, + processor: new (options: { processorOptions: PitchOptions }) => object, +): void; + +export interface PitchOptions { + /** The file the samples are of: they are sent with the first voice of it only and kept here. */ + file: string; + channels?: Float32Array[]; + /** The sound's rate over the context's. */ + step: number; + loop: boolean; +} + +/** Every sound a voice has been played from, for the next voice of it – processors share this scope. */ +const sounds = new Map(); + +class PitchProcessor extends AudioWorkletProcessor { + private readonly player: PitchPlayer; + private stopped = false; + + constructor({ processorOptions: { file, channels, step, loop } }: { processorOptions: PitchOptions }) { + super(); + if (channels) sounds.set(file, channels); + this.player = new PitchPlayer(sounds.get(file) ?? [], step, sampleRate, loop); + this.port.onmessage = () => { + this.stopped = true; + }; + } + + process( + _inputs: Float32Array[][], + outputs: Float32Array[][], + parameters: Record, + ): boolean { + if (this.stopped) return false; + if (this.player.render(outputs[0]!, parameters.pitch!)) return true; + this.port.postMessage('ended'); + return false; + } + + static get parameterDescriptors() { + return [{ name: 'pitch', defaultValue: 0 }]; + } +} + +registerProcessor('pitch-player', PitchProcessor); diff --git a/tests/core/pitch-player.test.ts b/tests/core/pitch-player.test.ts new file mode 100644 index 00000000..ee0c1afe --- /dev/null +++ b/tests/core/pitch-player.test.ts @@ -0,0 +1,140 @@ +/** + * The pitch of a sound FX in real time (#317): a sound played higher or lower keeps its length, also + * while the pitch moves. Measured on a sine, whose pitch is the number of times it rises through zero + * per second. + */ +import { describe, expect, it } from 'vitest'; +import { PitchPlayer } from '../../src/core/soundfx/pitchPlayer'; + +const RATE = 48000; +const BLOCK = 128; +/** A frame at 48 kHz is 2048 samples, and a frame takes the pitch at the sample it starts on. */ +const HALF_FRAME = 1024; + +function sine(frequency: number, seconds: number, rate = RATE): Float32Array { + const samples = new Float32Array(Math.round(rate * seconds)); + for (let i = 0; i < samples.length; i++) samples[i] = Math.sin((2 * Math.PI * frequency * i) / rate); + return samples; +} + +/** Plays the sound to its end block by block, as a worklet does, the pitch from `semitones(sample)`. */ +function render( + sound: Float32Array, + semitones: number | ((at: number) => number), + { step = 1, loop = false, blocks = Infinity } = {}, +): Float32Array { + const player = new PitchPlayer([sound], step, RATE, loop); + const out: number[] = []; + const block = new Float32Array(BLOCK); + const pitch = new Float32Array(BLOCK); + for (let b = 0; b < blocks; b++) { + for (let i = 0; i < BLOCK; i++) + pitch[i] = typeof semitones === 'number' ? semitones : semitones(out.length + i); + const going = player.render([block], typeof semitones === 'number' ? [semitones] : pitch); + out.push(...block); + if (!going) break; + } + return Float32Array.from(out); +} + +/** Rises through zero per second between two samples. */ +function frequencyOf( + samples: Float32Array, + from = samples.length / 4, + to = (samples.length * 3) / 4, +): number { + let rises = 0; + for (let i = Math.floor(from) + 1; i < to; i++) if (samples[i - 1]! < 0 && samples[i]! >= 0) rises++; + return rises / ((to - from) / RATE); +} + +const loudest = (samples: Float32Array) => samples.reduce((max, v) => Math.max(max, Math.abs(v)), 0); + +/** The last sample that is not silence, plus one: how long the sound is heard. */ +function heard(samples: Float32Array): number { + let last = samples.length; + while (last > 0 && Math.abs(samples[last - 1]!) <= 1e-4) last--; + return last; +} + +describe('the pitch player', () => { + it.each([ + [12, 880], + [-12, 220], + [7, 440 * 2 ** (7 / 12)], + [0.5, 440 * 2 ** (0.5 / 12)], + [-30.5, 440 * 2 ** (-30.5 / 12)], + ])('moves a 440 Hz tone by %d semitones to about %d Hz, at the same length', (semitones, expected) => { + const tone = sine(440, 1); + const shifted = render(tone, semitones); + expect(shifted.length).toBeGreaterThanOrEqual(tone.length); + expect(heard(shifted)).toBeLessThanOrEqual(tone.length); + expect(heard(shifted)).toBeGreaterThan(tone.length - 50); + expect(frequencyOf(shifted)).toBeGreaterThan(expected * 0.98); + expect(frequencyOf(shifted)).toBeLessThan(expected * 1.02); + expect(loudest(shifted)).toBeLessThan(1.2); + expect(loudest(shifted.subarray(RATE / 4, (RATE * 3) / 4))).toBeGreaterThan(0.7); + }); + + it('plays a sound at 0 semitones as it is, from the very first sample', () => { + const tone = sine(440, 0.5); + const same = render(tone, 0); + let off = 0; + for (let i = 0; i < tone.length; i++) off = Math.max(off, Math.abs(same[i]! - tone[i]!)); + expect(off).toBeLessThan(1e-5); + expect(heard(same)).toBe(tone.length); + }); + + it('starts the pitched sound at once rather than after a frame', () => { + const shifted = render(sine(440, 0.5), 12); + expect(loudest(shifted.subarray(0, 480))).toBeGreaterThan(0.3); + }); + + it('keeps the length while the pitch sweeps down, the pitch half a frame behind the sweep', () => { + // A blip of a quarter second taken down two octaves over its own length. + const tone = sine(880, 0.25); + const sweep = (at: number) => (-24 * at) / tone.length; + const swept = render(tone, sweep); + expect(heard(swept)).toBeLessThanOrEqual(tone.length); + expect(heard(swept)).toBeGreaterThan(tone.length - 50); + for (const at of [tone.length / 4, (tone.length * 3) / 4]) { + const expected = 880 * 2 ** (sweep(at - HALF_FRAME) / 12); + const heardThere = frequencyOf(swept, at - 1200, at + 1200); + expect(heardThere).toBeGreaterThan(expected * 0.95); + expect(heardThere).toBeLessThan(expected * 1.05); + } + }); + + it('reads a sound at its own rate: an 8 kHz tone comes out at its pitch and length', () => { + const tone = sine(440, 0.5, 8000); + const played = render(tone, 12, { step: 8000 / RATE }); + expect(heard(played)).toBeGreaterThan(RATE / 2 - 50); + expect(heard(played)).toBeLessThanOrEqual(RATE / 2); + expect(frequencyOf(played)).toBeGreaterThan(880 * 0.98); + expect(frequencyOf(played)).toBeLessThan(880 * 1.02); + }); + + it('reads on across the end of a looping sound, so its seam does not fade out, and never ends', () => { + const tone = sine(500, 0.25); + const looped = render(tone, 12, { loop: true, blocks: Math.ceil((tone.length * 2) / BLOCK) }); + for (let from = tone.length - 960; from < tone.length + 960; from += 96) + expect(loudest(looped.subarray(from, from + 96))).toBeGreaterThan(0.8); + expect(heard(looped)).toBe(looped.length); + expect(heard(render(tone, 12))).toBeLessThanOrEqual(tone.length); + }); + + it('goes from 0 semitones to a pitch and back without a dip, and then plays the sound as it is', () => { + const tone = sine(440, 2); + // Up to 5 over 0.1 s from a third in, back down over 0.1 s from two thirds. + const ramp = (at: number) => Math.max(0, Math.min(1, (at - RATE / 3) / (RATE / 10))); + const moved = render(tone, (at) => 5 * (ramp(at) - ramp(at - RATE / 3))); + for (let from = 0; from + 480 <= tone.length; from += 480) + expect(loudest(moved.subarray(from, from + 480))).toBeGreaterThan(0.8); + const up = frequencyOf(moved, RATE * 0.5, RATE * 0.6); + expect(up).toBeGreaterThan(440 * 2 ** (5 / 12) * 0.98); + expect(up).toBeLessThan(440 * 2 ** (5 / 12) * 1.02); + let off = 0; + for (let i = RATE * 1.5; i < RATE * 2 - 50; i++) off = Math.max(off, Math.abs(moved[i]! - tone[i]!)); + expect(off).toBeLessThan(1e-5); + }); +}); diff --git a/tests/core/pitch-shift.test.ts b/tests/core/pitch-shift.test.ts deleted file mode 100644 index 482c1801..00000000 --- a/tests/core/pitch-shift.test.ts +++ /dev/null @@ -1,56 +0,0 @@ -/** - * The pitch of a sound FX (#312): a sound played higher or lower keeps its length. Measured on a - * sine, whose pitch is the number of times it rises through zero per second. - */ -import { describe, expect, it } from 'vitest'; -import { pitchShift } from '../../src/core/soundfx/pitchShift'; - -const RATE = 16000; - -function sine(frequency: number, seconds: number): Float32Array { - const samples = new Float32Array(Math.round(RATE * seconds)); - for (let i = 0; i < samples.length; i++) samples[i] = Math.sin((2 * Math.PI * frequency * i) / RATE); - return samples; -} - -/** Rises through zero per second over the middle half, away from the fade at either end. */ -function frequencyOf(samples: Float32Array): number { - const from = Math.floor(samples.length / 4); - const to = Math.floor((samples.length * 3) / 4); - let rises = 0; - for (let i = from + 1; i < to; i++) if (samples[i - 1]! < 0 && samples[i]! >= 0) rises++; - return rises / ((to - from) / RATE); -} - -const loudest = (samples: Float32Array) => samples.reduce((max, v) => Math.max(max, Math.abs(v)), 0); - -describe('pitch shifting', () => { - it.each([ - [12, 880], - [-12, 220], - [7, 440 * 2 ** (7 / 12)], - [0.5, 440 * 2 ** (0.5 / 12)], - [-30.5, 440 * 2 ** (-30.5 / 12)], - ])('moves a 440 Hz tone by %d semitones to about %d Hz, at the same length', (semitones, expected) => { - const tone = sine(440, 1); - const shifted = pitchShift(tone, RATE, semitones); - expect(shifted.length).toBe(tone.length); - expect(frequencyOf(shifted)).toBeGreaterThan(expected * 0.98); - expect(frequencyOf(shifted)).toBeLessThan(expected * 1.02); - }); - - it('leaves a sound alone at 0 and does not make it louder', () => { - const tone = sine(440, 0.5); - const same = pitchShift(tone, RATE, 0); - const middle = tone.length / 2; - expect(same[middle]).toBeCloseTo(tone[middle]!, 5); - expect(loudest(pitchShift(tone, RATE, 12))).toBeLessThanOrEqual(1.0001); - }); - - it('reads on across the end of a looping sound, so its seam does not fade out', () => { - const tone = sine(500, 0.25); - const tail = (samples: Float32Array) => loudest(samples.subarray(samples.length - 40)); - expect(tail(pitchShift(tone, RATE, 12))).toBeLessThan(0.6); - expect(tail(pitchShift(tone, RATE, 12, true))).toBeGreaterThan(0.95); - }); -}); diff --git a/tests/core/soundfx.test.ts b/tests/core/soundfx.test.ts index daacf501..dc2eeec5 100644 --- a/tests/core/soundfx.test.ts +++ b/tests/core/soundfx.test.ts @@ -176,7 +176,7 @@ describe('compiling a sound FX graph', () => { const aside = f.edit.addNode('SoundFxGain', { gain: -20 }); f.edit.connect(f.input, 'out', aside, 'in'); const build = compileSoundFx(f.fx.graph); - expect(fxChainOf(build.compiled!)).toEqual({ pitch: 3, stages: [] }); + expect(fxChainOf(build.compiled!)).toEqual({ pitch: 3, shifted: true, stages: [] }); expect(build.effects).toBe(1); }); }); @@ -356,11 +356,18 @@ describe('the parameters of a sound FX', () => { const f = withParam('SoundFxGain', 'gain', [{ key: 'loud', kind: 'number' }]); const plan = soundFxBuildFor(f.fx).compiled!; expect(plan.stages).toHaveLength(1); - expect(fxChainOf(plan)).toEqual({ pitch: 0, stages: [{ kind: 'gain', gain: 1 }] }); + expect(fxChainOf(plan)).toEqual({ pitch: 0, shifted: false, stages: [{ kind: 'gain', gain: 1 }] }); expect(fxChainOf(plan, { loud: 6 }).stages[0]).toMatchObject({ kind: 'gain' }); expect((fxChainOf(plan, { loud: 6 }).stages[0] as { gain: number }).gain).toBeCloseTo(10 ** (6 / 20)); }); + it('keep a pitch that is 0 by default on the worklet, and only a typed-in 0 off it (#317)', () => { + const fed = withParam('SoundFxPitch', 'semitones', [{ key: 'depth', kind: 'number' }]); + expect(fxChainOf(soundFxBuildFor(fed.fx).compiled!)).toMatchObject({ pitch: 0, shifted: true }); + const typed = line([{ type: 'SoundFxPitch', params: { semitones: 0 } }]); + expect(fxChainOf(soundFxBuildFor(typed.fx).compiled!)).toMatchObject({ pitch: 0, shifted: false }); + }); + it('feed an EQ band as well', () => { const f = withParam('SoundFxEq', 'mid', [{ key: 'body', kind: 'number', default: 3 }]); const chain = fxChainOf(soundFxBuildFor(f.fx).compiled!, { body: -4 }); @@ -474,7 +481,7 @@ describe('Sound FX parameters and Play sound', () => { ]); const audio = fakeAudio(); startWorld(f, { behaviors: true, audio }).frame(); - expect(audio.played[0]!.fx).toEqual({ pitch: -12, stages: [{ kind: 'gain', gain: 1 }] }); + expect(audio.played[0]!.fx).toEqual({ pitch: -12, shifted: true, stages: [{ kind: 'gain', gain: 1 }] }); }); it('says so at the node when no FX is picked or it is gone', () => { @@ -500,7 +507,7 @@ describe('Sound FX parameters and Play sound', () => { expect(f.params.params.drop).toBe(7); const audio = fakeAudio(); startWorld(f, { behaviors: true, audio }).frame(); - expect(audio.played[0]!.fx).toEqual({ pitch: 7, stages: [{ kind: 'gain', gain: 1 }] }); + expect(audio.played[0]!.fx).toEqual({ pitch: 7, shifted: true, stages: [{ kind: 'gain', gain: 1 }] }); f.stack.undo(); expect(f.params.ports).toEqual([{ key: 'depth', kind: 'number' }]); expect(f.params.params.depth).toBe(7); diff --git a/tests/viewport/sound-library.test.ts b/tests/viewport/sound-library.test.ts index 970af6f8..b0b63a8f 100644 --- a/tests/viewport/sound-library.test.ts +++ b/tests/viewport/sound-library.test.ts @@ -6,9 +6,15 @@ * The device is stubbed – nothing makes a noise in a unit test, and Web Audio does not exist here. */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { FxChain, FxPlan, FxVoice } from '../../src/core/soundfx/nodes'; +import type { FxPlan, FxStage, FxVoice } from '../../src/core/soundfx/nodes'; +import type { PitchOptions } from '../../src/viewport/pitchWorklet'; import { SoundLibrary } from '../../src/viewport/SoundLibrary'; +/** What a test wants to hear: a chain whose pitch, where it is not 0, is a Pitch typed in. */ +type Chain = + | { pitch: number; stages: readonly FxStage[] } + | { mix: number; a: Chain; b: Chain; stages: readonly FxStage[] }; + const FILE = 'a.wav'; const OTHER = 'b.wav'; @@ -60,6 +66,23 @@ interface FakeSource { stop: () => void; } +/** The pitch worklet of a way (#317): what it was made with, its pitch and what was posted to it. */ +interface FakeWorklet { + options: { outputChannelCount: number[]; parameterData: { pitch: number }; processorOptions: PitchOptions }; + pitch: FakeParam; + parameters: Map; + posted: unknown[]; + port: { postMessage: (message: unknown) => void; onmessage: ((event: { data: unknown }) => void) | null }; + disconnected: boolean; + connect: (target: unknown) => unknown; + disconnect: () => void; +} + +const worklets: FakeWorklet[] = []; +/** Every module loaded into a context, and what the next load does. */ +const modules: string[] = []; +let loadModule: () => Promise = () => Promise.resolve(); + const sources: FakeSource[] = []; /** Every `connect` made, as "from > to" by the kind of node – the order a voice is wired in. */ const wires: string[] = []; @@ -106,6 +129,13 @@ function fakeContext() { return { state: 'running', currentTime: 0, + sampleRate: 48000, + audioWorklet: { + addModule: (url: string) => { + modules.push(url); + return loadModule(); + }, + }, destination: { kind: 'destination' }, resume: () => Promise.resolve(), suspend: () => Promise.resolve(), @@ -152,6 +182,9 @@ const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); beforeEach(() => { sources.length = 0; + worklets.length = 0; + modules.length = 0; + loadModule = () => Promise.resolve(); wires.length = 0; filters.length = 0; gains.length = 0; @@ -164,6 +197,30 @@ beforeEach(() => { } }, ); + vi.stubGlobal( + 'AudioWorkletNode', + class { + constructor(_context: unknown, _name: string, options: FakeWorklet['options']) { + const pitch = param(options.parameterData.pitch); + const node: FakeWorklet = wired('pitch', { + options, + pitch, + parameters: new Map([['pitch', pitch]]), + posted: [] as unknown[], + port: { + postMessage: (message: unknown) => node.posted.push(message), + onmessage: null, + }, + disconnected: false, + disconnect: () => { + node.disconnected = true; + }, + }); + worklets.push(node); + return node; + } + }, + ); vi.stubGlobal('fetch', () => Promise.resolve({ ok: true, arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)) }), ); @@ -174,7 +231,7 @@ afterEach(() => { }); /** A chain as the plan of an FX with nothing left open, so a test can say what it wants to hear. */ -function planOf(chain: FxChain): FxPlan { +function planOf(chain: Chain): FxPlan { const stages = chain.stages.map((stage) => stage.kind === 'gain' ? { kind: 'gain' as const, db: { value: 20 * Math.log10(stage.gain) } } @@ -184,12 +241,22 @@ function planOf(chain: FxChain): FxPlan { return { pitch: chain.pitch === 0 ? [] : [{ value: chain.pitch }], stages }; } +/** Pitch on a, a gain on b, the share between them – all three open – and a band after the Mix. */ +function open(): FxPlan { + return { + mix: { value: 0.25, param: 'blend', min: 0, max: 1 }, + a: { pitch: [{ value: 0, param: 'depth', min: -64, max: 64 }], stages: [] }, + b: { pitch: [], stages: [{ kind: 'gain', db: { value: 0, param: 'loud', min: -120, max: 24 } }] }, + stages: [{ kind: 'filter', filter: 'lowshelf', frequency: 80, gain: { value: 6 }, q: 1 }], + }; +} + /** Plays `file` and lets its bytes arrive, unless `hold` keeps the decode pending; gives the voice. */ async function play( library: SoundLibrary, file = FILE, { hold = false, owner = '' } = {}, - fx?: FxChain | FxVoice, + fx?: Chain | FxVoice, ): Promise { const voice = library.play( file, @@ -267,15 +334,16 @@ describe('the sound library', () => { }); }); - it('plays a dry voice straight into its volume', async () => { + it('plays a dry voice straight into its volume, without loading the pitch worklet', async () => { await play(new SoundLibrary()); expect(wires).toEqual(['source > gain', 'gain > destination']); expect(sources[0]?.playbackRate.value).toBe(1); + expect(modules).toEqual([]); }); it('sends a voice through its sound FX in order (#311)', async () => { const library = new SoundLibrary(); - const fx: FxChain = { + const fx: Chain = { pitch: 12, stages: [ { kind: 'gain', gain: 0.5 }, @@ -283,25 +351,59 @@ describe('the sound library', () => { ], }; await play(library, FILE, {}, fx); - expect(wires).toEqual(['source > gain', 'gain > filter', 'filter > gain', 'gain > destination']); + expect(wires).toEqual(['pitch > gain', 'gain > filter', 'filter > gain', 'gain > destination']); expect(filters).toMatchObject([ { type: 'lowshelf', frequency: { value: 80 }, gain: { value: 6 }, Q: { value: 1 } }, ]); expect(library.stats().last).toMatchObject({ pitches: [12], stages: 2 }); }); - it('pitches a voice on a copy of its length rather than by speed, and keeps the copy (#312)', async () => { + it('plays a pitched way through the worklet at the length of the sound, its samples sent once (#317)', async () => { const library = new SoundLibrary(); await play(library); await play(library, FILE, {}, { pitch: 7, stages: [] }); - await play(library, FILE, {}, { pitch: 7, stages: [] }); await play(library, FILE, {}, { pitch: -3.5, stages: [] }); - const [dry, up, again, down] = sources.map((s) => s.buffer as { length: number; duration: number }); - expect(sources.map((s) => s.playbackRate.value)).toEqual([1, 1, 1, 1]); - expect(up).not.toBe(dry); - expect(again).toBe(up); - expect(down).not.toBe(up); - expect([up!.duration, down!.duration]).toEqual([dry!.duration, dry!.duration]); + await play(library, OTHER, {}, { pitch: 7, stages: [] }); + expect(modules).toHaveLength(1); + expect(sources).toHaveLength(1); + expect(worklets.map((w) => w.options.parameterData.pitch)).toEqual([7, -3.5, 7]); + expect(worklets.map((w) => w.options.processorOptions.file)).toEqual([FILE, FILE, OTHER]); + // The fake decodes at 800 Hz into a context at 48 kHz. + expect(worklets[0]!.options).toMatchObject({ + outputChannelCount: [1], + processorOptions: { step: 800 / 48000, loop: false }, + }); + expect(worklets.map((w) => w.options.processorOptions.channels?.[0]?.length)).toEqual([ + 1200, + undefined, + 1200, + ]); + expect(library.stats().last).toMatchObject({ pitches: [7], duration: 1.5 }); + }); + + it('a play that needs the worklet waits for it to load, as it would for its bytes', async () => { + let loaded = () => {}; + loadModule = () => new Promise((resolve) => (loaded = resolve)); + const library = new SoundLibrary(); + await play(library); + await play(library, FILE, {}, { pitch: 7, stages: [] }); + expect(library.stats()).toMatchObject({ started: 1 }); + loaded(); + await settle(); + expect(library.stats()).toMatchObject({ started: 2, last: { pitches: [7] } }); + expect(worklets).toHaveLength(1); + }); + + it('without the worklet a pitched way plays at its own pitch rather than not at all', async () => { + loadModule = () => Promise.reject(new Error('no worklets here')); + vi.spyOn(console, 'error').mockImplementation(() => {}); + const library = new SoundLibrary(); + const voice = await play(library, FILE, {}, { plan: open(), params: {} }); + expect(worklets).toHaveLength(0); + expect(sources).toHaveLength(2); + expect(library.tune(voice, { depth: 12 })).toBe(true); + expect(library.levels(FILE)).toEqual([[12, 0.75, 1, 0.25, 6]]); + expect(console.error).toHaveBeenCalledTimes(1); }); it('mixes two sounds by their shares, a source per way, started and stopped together (#316)', async () => { @@ -318,7 +420,7 @@ describe('the sound library', () => { }, ); expect(wires).toEqual([ - 'source > gain', + 'pitch > gain', 'gain > gain', 'source > gain', 'gain > gain', @@ -328,16 +430,26 @@ describe('the sound library', () => { 'gain > destination', ]); expect(gains.map((g) => g.value)).toEqual([1, 1, 0.75, expect.closeTo(0.5), 0.25]); - expect(sources.map((s) => s.started)).toEqual([true, true]); + expect(sources.map((s) => s.started)).toEqual([true]); expect(library.stats()).toMatchObject({ playing: 1, started: 1, last: { pitches: [12, 0], stages: 2 } }); - sources[0]!.onended!(); + worklets[0]!.port.onmessage!({ data: 'ended' }); expect(library.stats().playing).toBe(1); library.stop(FILE); - expect(sources.map((s) => s.stopped)).toEqual([true, true]); + expect(sources.map((s) => s.stopped)).toEqual([true]); + expect(worklets[0]).toMatchObject({ posted: ['stop'], disconnected: true }); expect(library.stats()).toMatchObject({ playing: 0, stopped: 1 }); }); + it('a pitched voice that ends by itself leaves the count when the worklet says so', async () => { + const library = new SoundLibrary(); + await play(library, FILE, {}, { pitch: 3, stages: [] }); + worklets[0]!.port.onmessage!({ data: 'something else' }); + expect(library.stats().playing).toBe(1); + worklets[0]!.port.onmessage!({ data: 'ended' }); + expect(library.stats()).toMatchObject({ playing: 0, stopped: 0 }); + }); + it('gives two ways at the same pitch a source each, so each can glide on its own (#315)', async () => { const library = new SoundLibrary(); await play( @@ -357,24 +469,16 @@ describe('the sound library', () => { }); describe('a parameter changed while the voice plays (#315)', () => { - /** Pitch on a, a gain on b, the share between them – all three open – and a band after the Mix. */ - const open: FxPlan = { - mix: { value: 0.25, param: 'blend', min: 0, max: 1 }, - a: { pitch: [{ value: 0, param: 'depth', min: -64, max: 64 }], stages: [] }, - b: { pitch: [], stages: [{ kind: 'gain', db: { value: 0, param: 'loud', min: -120, max: 24 } }] }, - stages: [{ kind: 'filter', filter: 'lowshelf', frequency: 80, gain: { value: 6 }, q: 1 }], - }; - - it('glides every knob it moves, a pitch by speed, and leaves the rest alone', async () => { + it('glides every knob it moves, a pitch in semitones, and leaves the rest alone', async () => { const library = new SoundLibrary(); - const voice = await play(library, FILE, { owner: 'me' }, { plan: open, params: {} }); + const voice = await play(library, FILE, { owner: 'me' }, { plan: open(), params: {} }); // Made in the order: the voice, the Mix, share of a, the gain on b, share of b. const [, , shareA, loud, shareB] = gains; expect([shareA!.value, loud!.value, shareB!.value]).toEqual([0.75, 1, 0.25]); expect(library.tune(voice, { depth: 12, loud: -6, blend: 0.5, nothing: 3 })).toBe(true); - expect(sources[0]!.playbackRate.glides).toEqual([2]); - expect(sources[1]!.playbackRate.glides).toEqual([]); + expect(worklets[0]!.pitch.glides).toEqual([12]); + expect(sources[0]!.playbackRate.glides).toEqual([]); expect(loud!.glides).toEqual([expect.closeTo(10 ** (-6 / 20))]); expect([shareA!.glides, shareB!.glides]).toEqual([[0.5], [0.5]]); expect(filters[0]!.gain.glides).toEqual([]); @@ -382,7 +486,7 @@ describe('the sound library', () => { // Only what changed is sent again; one not given keeps its value rather than its default. library.tune(voice, { depth: 12, blend: 1 }); - expect(sources[0]!.playbackRate.glides).toEqual([2]); + expect(worklets[0]!.pitch.glides).toEqual([12]); expect(loud!.glides).toHaveLength(1); expect([shareA!.glides, shareB!.glides]).toEqual([ [0.5, 0], @@ -393,17 +497,16 @@ describe('the sound library', () => { it('reaches the one voice it is given – a detached one too – and none that has ended', async () => { const library = new SoundLibrary(); const voices = [ - await play(library, FILE, { owner: 'me' }, { plan: open, params: {} }), - await play(library, FILE, { owner: 'me' }, { plan: open, params: {} }), - await play(library, OTHER, { owner: 'me' }, { plan: open, params: {} }), - await play(library, FILE, {}, { plan: open, params: {} }), + await play(library, FILE, { owner: 'me' }, { plan: open(), params: {} }), + await play(library, FILE, { owner: 'me' }, { plan: open(), params: {} }), + await play(library, OTHER, { owner: 'me' }, { plan: open(), params: {} }), + await play(library, FILE, {}, { plan: open(), params: {} }), ]; expect(new Set(voices).size).toBe(4); expect(voices).not.toContain(0); library.tune(voices[1]!, { depth: 12 }); library.tune(voices[3]!, { depth: 12 }); - // Two sources a voice, the first of each the pitched way. - expect(sources.map((s) => s.playbackRate.glides.length)).toEqual([0, 0, 1, 0, 0, 0, 1, 0]); + expect(worklets.map((w) => w.pitch.glides.length)).toEqual([0, 1, 0, 1]); library.stop(FILE); expect(library.tune(voices[1]!, { depth: 3 })).toBe(false); @@ -417,59 +520,61 @@ describe('the sound library', () => { library, FILE, { hold: true, owner: 'me' }, - { plan: open, params: { blend: 0.5 } }, + { plan: open(), params: { blend: 0.5 } }, ); expect(library.tune(voice, { depth: 7 })).toBe(true); await arrive(); expect(library.stats().last?.pitches).toEqual([7, 0]); expect(gains.map((g) => g.value)).toContain(0.5); - expect(sources[0]!.playbackRate.glides).toEqual([]); + expect(worklets[0]!.options.parameterData.pitch).toBe(7); + expect(worklets[0]!.pitch.glides).toEqual([]); }); it('runs a timed change evenly on the audio clock: semitones and decibels in their own steps', async () => { const library = new SoundLibrary(); - const voice = await play(library, FILE, { owner: 'me' }, { plan: open, params: {} }); + const voice = await play(library, FILE, { owner: 'me' }, { plan: open(), params: {} }); const [, , shareA, loud, shareB] = gains; library.tune(voice, { depth: 12, loud: -6, blend: 0.5 }, 1.5); - expect(sources[0]!.playbackRate.ramps).toEqual([['exponential', 2, 1.5]]); + const pitch = worklets[0]!.pitch; + expect(pitch.ramps).toEqual([['linear', 12, 1.5]]); expect(loud!.ramps).toEqual([['exponential', expect.closeTo(10 ** (-6 / 20)), 1.5]]); expect([shareA!.ramps, shareB!.ramps]).toEqual([[['linear', 0.5, 1.5]], [['linear', 0.5, 1.5]]]); - expect([sources[0]!.playbackRate.glides, loud!.glides, shareA!.glides]).toEqual([[], [], []]); + expect([pitch.glides, loud!.glides, shareA!.glides]).toEqual([[], [], []]); // Each from where it is now: what was planned before is held, not queued behind. - expect([sources[0]!.playbackRate.holds, loud!.holds, shareA!.holds]).toEqual([1, 1, 1]); + expect([pitch.holds, loud!.holds, shareA!.holds]).toEqual([1, 1, 1]); // A quick change after it holds the ramp where it is and glides on from there. library.tune(voice, { depth: 0 }); - expect(sources[0]!.playbackRate.holds).toBe(2); - expect(sources[0]!.playbackRate.glides).toEqual([1]); + expect(pitch.holds).toBe(2); + expect(pitch.glides).toEqual([0]); }); it('runs a timed change along an easing as a curve, each number in its own steps', async () => { const library = new SoundLibrary(); - const voice = await play(library, FILE, { owner: 'me' }, { plan: open, params: {} }); + const voice = await play(library, FILE, { owner: 'me' }, { plan: open(), params: {} }); const loud = gains[3]!; const from = loud.value; library.tune(voice, { depth: 12, loud: -6 }, 1.5, (t) => t * t); // Half the time is a quarter of the way: a quarter of the octave, a quarter of the decibels. - expect(sources[0]!.playbackRate.ramps).toEqual([['curve', [1, expect.closeTo(2 ** 0.25), 2], 1.5]]); + expect(worklets[0]!.pitch.ramps).toEqual([['curve', [0, expect.closeTo(3), 12], 1.5]]); const quarter = from * (10 ** (-6 / 20) / from) ** 0.25; expect(loud.ramps).toEqual([ ['curve', [from, expect.closeTo(quarter), expect.closeTo(10 ** (-6 / 20))], 1.5], ]); - expect(sources[0]!.playbackRate.holds).toBe(1); + expect(worklets[0]!.pitch.holds).toBe(1); }); it('a step holds the value for the time and then jumps', async () => { const library = new SoundLibrary(); - const voice = await play(library, FILE, { owner: 'me' }, { plan: open, params: {} }); + const voice = await play(library, FILE, { owner: 'me' }, { plan: open(), params: {} }); library.tune(voice, { depth: 12 }, 2, 'step'); - expect(sources[0]!.playbackRate.ramps).toEqual([['set', 2, 2]]); - expect(sources[0]!.playbackRate.glides).toEqual([]); + expect(worklets[0]!.pitch.ramps).toEqual([['set', 12, 2]]); + expect(worklets[0]!.pitch.glides).toEqual([]); }); it('starts a timed change to a voice still waiting on its bytes when the voice starts', async () => { const library = new SoundLibrary(); - const voice = await play(library, FILE, { hold: true, owner: 'me' }, { plan: open, params: {} }); + const voice = await play(library, FILE, { hold: true, owner: 'me' }, { plan: open(), params: {} }); library.tune(voice, { depth: -64 }, 1.28); // A quick change after a timed one keeps its place behind it. library.tune(voice, { blend: 1 }); @@ -477,7 +582,7 @@ describe('the sound library', () => { await arrive(); // The voice is heard at its old pitch and only then sets off, so the whole sweep is heard. expect(library.stats().last?.pitches).toEqual([0, 0]); - expect(sources[0]!.playbackRate.ramps).toEqual([['exponential', 2 ** (-64 / 12), 1.28]]); + expect(worklets[0]!.pitch.ramps).toEqual([['linear', -64, 1.28]]); expect(gains[4]!.glides).toEqual([1]); }); }); diff --git a/tools/e2e/34-sound.mjs b/tools/e2e/34-sound.mjs index 609ee505..9fed9bc7 100644 --- a/tools/e2e/34-sound.mjs +++ b/tools/e2e/34-sound.mjs @@ -416,7 +416,8 @@ export async function sound(t) { await stop(); // #315: two looping voices of the same sound start at depth 7, the first keeps its number in a // variable, and `Set sound FX parameters` hands that one voice depth 3 every frame. In a - // real AudioContext the first one's speed glides there, the second one's stays put. + // real AudioContext the first one's pitch – the AudioParam of its worklet (#317), in + // semitones – glides there, the second one's stays put. const tuneIds = ['sn-keep', 'sn-play2', 'sn-tick', 'sn-read', 'sn-tune', 'sn-fxp2']; const tuneEdges = [ 'sn-e-keep', @@ -460,52 +461,52 @@ export async function sound(t) { stats, (s) => s.tuned > beforeTune.tuned && s.started >= beforeTune.started + 2, ); - const rate = 2 ** ((3 - 7) / 12); const glided = (levels) => - Array.isArray(levels) && levels.length === 2 && Math.abs(levels[0][0] - rate) < 0.01; + Array.isArray(levels) && levels.length === 2 && Math.abs(levels[0][0] - 3) < 0.05; const levels = await waitFor(() => json(`__gm.sound.levels(${JSON.stringify(blip.file)})`), glided); check( 'Set sound FX parameters moves the one voice whose number it holds from depth 7 to 3, not the other', tunedStats.last?.pitches.join() === '7' && tunedStats.last.loop === true && + tunedStats.worklet === true && glided(levels) && Math.abs(levels[0][1] - 10 ** (-6 / 20)) < 0.001 && - levels[1][0] === 1, - JSON.stringify({ last: tunedStats.last, tuned: tunedStats.tuned, levels, rate }), + levels[1][0] === 7, + JSON.stringify({ last: tunedStats.last, tuned: tunedStats.tuned, worklet: tunedStats.worklet, levels }), ); await stop(); - // A `time` of 2 s takes the same voice from depth 7 to -5 evenly on the audio clock: half a - // second in it is still well above the end (speed 0.5), where a quick glide would be there by then. + // A `time` of 2 s takes the same voice from depth 7 to -5 evenly on the audio clock, 6 semitones + // a second: half a second in it is still well above the end, where a quick glide would be there. await mutate(` const graph = __gm.state.project.assets.find((a) => a.id === 'sn-logic').graph; graph.nodes.find((n) => n.id === 'sn-tune').params.time = 2; graph.nodes.find((n) => n.id === 'sn-fxp2').params.depth = -5;`); await play(); - const speed = async () => (await json(`__gm.sound.levels(${JSON.stringify(blip.file)})`))?.[0]?.[0]; - const early = await waitFor(speed, (s) => typeof s === 'number' && s < 0.99); - const later = await waitFor(speed, (s) => typeof s === 'number' && s < early - 0.05); + const pitch = async () => (await json(`__gm.sound.levels(${JSON.stringify(blip.file)})`))?.[0]?.[0]; + const early = await waitFor(pitch, (s) => typeof s === 'number' && s < 6.9); + const later = await waitFor(pitch, (s) => typeof s === 'number' && s < early - 0.6); await new Promise((resolve) => setTimeout(resolve, 500)); - const afterHalf = await speed(); + const afterHalf = await pitch(); check( 'Set sound FX parameters with a time ramps the pitch over that time instead of jumping', - early > 0.6 && later > 0.55 && afterHalf > 0.52 && afterHalf < later, + early > -1.8 && later > -3.3 && afterHalf > -4.3 && afterHalf < later, JSON.stringify({ early, later, afterHalf }), ); await stop(); // The easings of an animation key: Quad in hands the clock a curve, which is still near the start - // 0.7 s in and only then falls to the end (speed 0.5). Read a little later than that for the - // round trips, it is above 0.8 until 1.1 s; linear is below 0.79 from 0.7 s on. + // 0.7 s in and only then falls to the end (-5). Read a little later than that for the round + // trips, it is above 3.1 until 1.1 s; linear is below 2.8 from 0.7 s on. await mutate(` const tune = __gm.state.project.assets.find((a) => a.id === 'sn-logic').graph.nodes.find((n) => n.id === 'sn-tune'); Object.assign(tune.params, { ease: 'quad', direction: 'in' });`); await play(); - await waitFor(speed, (s) => typeof s === 'number'); + await waitFor(pitch, (s) => typeof s === 'number'); await new Promise((resolve) => setTimeout(resolve, 700)); - const easedEarly = await speed(); - const easedEnd = await waitFor(speed, (s) => typeof s === 'number' && s < 0.52); + const easedEarly = await pitch(); + const easedEnd = await waitFor(pitch, (s) => typeof s === 'number' && s < -4.3); check( 'Set sound FX parameters with Quad in ramps slowly first, then on to the end', - easedEarly > 0.8 && easedEarly < 0.995 && easedEnd < 0.52, + easedEarly > 3.1 && easedEarly < 6.95 && easedEnd < -4.3, JSON.stringify({ easedEarly, easedEnd }), ); await stop(); @@ -514,16 +515,75 @@ export async function sound(t) { const tune = __gm.state.project.assets.find((a) => a.id === 'sn-logic').graph.nodes.find((n) => n.id === 'sn-tune'); Object.assign(tune.params, { ease: 'step', time: 1 });`); await play(); - await waitFor(speed, (s) => typeof s === 'number'); + await waitFor(pitch, (s) => typeof s === 'number'); await new Promise((resolve) => setTimeout(resolve, 500)); - const stepHeld = await speed(); - const stepEnd = await waitFor(speed, (s) => typeof s === 'number' && s < 0.9); + const stepHeld = await pitch(); + const stepEnd = await waitFor(pitch, (s) => typeof s === 'number' && s < 6); check( 'Set sound FX parameters with Step holds the pitch for the time, then jumps to the end', - stepHeld === 1 && Math.abs(stepEnd - 0.5) < 0.001, + stepHeld === 7 && Math.abs(stepEnd + 5) < 0.001, JSON.stringify({ stepHeld, stepEnd }), ); await stop(); + // #317: the worklet itself, rendered in an offline context of this browser – what a muted check + // can hear. A second of 440 Hz an octave up is 880 Hz and still a second long; a sweep from + // 0 to -12 over it keeps that second too. Then 16 stereo voices at +5 for 2 s, timed, for + // how many this machine plays at once. + const offline = await evaluate(` + (async () => { + const RATE = 48000; + const render = async (voices, seconds, channels, shape) => { + const context = new OfflineAudioContext(channels, RATE * seconds, RATE); + await context.audioWorklet.addModule(__gm.sound.pitchWorkletUrl()); + const tone = new Float32Array(RATE * seconds); + for (let i = 0; i < tone.length; i++) tone[i] = Math.sin((2 * Math.PI * 440 * i) / RATE) * 0.5; + for (let v = 0; v < voices; v++) { + const node = new AudioWorkletNode(context, 'pitch-player', { + numberOfInputs: 0, + outputChannelCount: [channels], + parameterData: { pitch: shape.from }, + processorOptions: { file: 'probe', channels: v === 0 ? Array(channels).fill(tone) : undefined, step: 1, loop: false }, + }); + if (shape.to !== shape.from) + node.parameters.get('pitch').setValueAtTime(shape.from, 0).linearRampToValueAtTime(shape.to, seconds); + node.connect(context.destination); + } + const started = performance.now(); + const out = (await context.startRendering()).getChannelData(0); + return { out, ms: performance.now() - started }; + }; + const rises = (out, from, to) => { + let count = 0; + for (let i = from + 1; i < to; i++) if (out[i - 1] < 0 && out[i] >= 0) count++; + return count / ((to - from) / RATE); + }; + const heard = (out) => { + let last = out.length; + while (last > 0 && Math.abs(out[last - 1]) <= 1e-4) last--; + return last; + }; + const up = await render(1, 1, 1, { from: 12, to: 12 }); + const sweep = await render(1, 1, 1, { from: 0, to: -12 }); + const load = await render(16, 2, 2, { from: 5, to: 5 }); + return { + upHz: rises(up.out, 12000, 36000), + upLength: heard(up.out), + sweepEarlyHz: rises(sweep.out, 6000, 12000), + sweepLateHz: rises(sweep.out, 36000, 42000), + sweepLength: heard(sweep.out), + voices: 16, + renderMs: Math.round(load.ms), + realTimeVoices: Math.round((16 * 2000) / load.ms), + }; + })()`); + check( + 'the pitch worklet plays an octave up at the length of the sound, and a sweep down keeps it too', + Math.abs(offline.upHz / 880 - 1) < 0.02 && + offline.upLength > 48000 - 100 && + offline.sweepEarlyHz > offline.sweepLateHz * 1.3 && + offline.sweepLength > 48000 - 100, + JSON.stringify(offline), + ); await mutate(` const logic = __gm.state.project.assets.find((a) => a.id === 'sn-logic'); logic.variables = [];