diff --git a/README.md b/README.md index 3565b418..b9d19b3e 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh The browser suite and the three example games need a machine with a GPU: ```bash -node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 1922 checks (GM_STEPS=20 for one step) +node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 1928 checks (GM_STEPS=20 for one step) # every tool signs in first – see the Accounts page node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs diff --git a/src/core/model/Asset.ts b/src/core/model/Asset.ts index d21586e9..40a23b0b 100644 --- a/src/core/model/Asset.ts +++ b/src/core/model/Asset.ts @@ -1,4 +1,5 @@ import type { Graph } from '../graph/Graph'; +import type { Bake } from '../music/bake'; import type { PatternNote } from '../music/pattern'; import type { SongItem, SongTrack } from '../music/song'; import type { LoopRange } from '../music/sequence'; @@ -357,6 +358,8 @@ export interface PatternAsset extends AssetBase { type: 'pattern'; notes: PatternNote[]; loop?: LoopRange; + /** What it was last baked to (#340); it plays only while nothing it is made of has changed since. */ + bake?: Bake; } /** @@ -370,6 +373,7 @@ export interface SongAsset extends AssetBase { items: SongItem[]; loop?: LoopRange; graph: Graph; + bake?: Bake; } /** diff --git a/src/core/model/projectFiles.ts b/src/core/model/projectFiles.ts index aa49e5a2..f73a5e51 100644 --- a/src/core/model/projectFiles.ts +++ b/src/core/model/projectFiles.ts @@ -1,3 +1,4 @@ +import { validBake } from '../music/bake'; import { isFontFile } from './font'; import { celFilesOf } from './pixelart'; import type { Project } from './Project'; @@ -15,7 +16,8 @@ export interface ProjectFiles { /** * Every file an asset names, used or not: which frame an animation or a tile reaches next is only * known while it plays, and a file fetched then shows as a gap. A name that is no stored file – - * an upload that failed – is left out, as the libraries would never get it anyway. + * an upload that failed – is left out, as the libraries would never get it anyway. So is a bake that + * no longer holds (#340): it would never be played, and decoded it weighs as much as the music. */ export function projectFiles(project: Project): ProjectFiles { const files: ProjectFiles = { textures: [], fonts: [], cels: [], sounds: [] }; @@ -24,6 +26,8 @@ export function projectFiles(project: Project): ProjectFiles { else if (asset.type === 'font' && isFontFile(asset.file)) files.fonts.push(asset.file); else if (asset.type === 'pixelart') files.cels.push(...celFilesOf(asset)); else if (asset.type === 'sound' && isSoundFile(asset.file)) files.sounds.push(asset.file); + else if ((asset.type === 'pattern' || asset.type === 'song') && validBake(project, asset)) + files.sounds.push(asset.bake!.file); } return files; } diff --git a/src/core/music/bake.ts b/src/core/music/bake.ts new file mode 100644 index 00000000..24f7f798 --- /dev/null +++ b/src/core/music/bake.ts @@ -0,0 +1,252 @@ +/** + * A pattern or a song baked (#340): what it plays worked out once, kept as Opus in Ogg, and played + * from there by `Play pattern` and `Play song` for as long as nothing it is made of has changed. + * Everything after its output stays live: the sound FX of the node, its volume, its bus. + * + * The file holds two parts, counted in frames at `BAKE_RATE`: the playing straight from the start + * (`head` long, as far as the end or the loop end, whichever is later) and then one more round of the + * loop as it is heard the second time round, with what rang on from the first in it. Played once, a + * bake runs from 0 to `length`; looped, it runs to `loopEnd` and then round its second round for good. + */ +import { instrumentBuildFor, instrumentSourcesOf } from '../instrument/compile'; +import { heldNote } from '../instrument/notes'; +import type { PatternAsset, SongAsset } from '../model/Asset'; +import { findAssetOf, type Project } from '../model/Project'; +import { findSound, isSoundFile } from '../model/sound'; +import { pickedFx } from '../properties/assetFx'; +import { playedSoundOf } from '../properties/playedSound'; +import { resolveAssetProps } from '../properties/resolve'; +import { volumeOf } from '../properties/sounds'; +import { clipFx, fxVoiceOf } from './needs'; +import { OPUS_RATE } from './ogg'; +import { patternLength, patternSequence, patternTempo } from './pattern'; +import type { BusLevel } from './playbacks'; +import type { LoopRange, Sequence } from './sequence'; +import { songLength, songLoop, songSequence, songTempo } from './song'; +import { songBuses } from './songBuses'; +import { secondsOfTicks } from './time'; + +export const BAKE_RATE = OPUS_RATE; +/** Goes up whenever the way a playing sounds changes, so every bake made before is outdated. */ +const BAKE_VERSION = 1; + +/** What a pattern or song keeps of its bake. Frames are at `BAKE_RATE`. */ +export interface Bake { + /** The Ogg file in the sounds of its owner (#344). */ + file: string; + /** What it was baked from (`bakePrint`): it plays only while that is still the same. */ + print: string; + /** The loudest sample before it was raised to full scale; what it is played at. */ + peak: number; + length: number; + loopEnd: number; + head: number; + round: number; + /** The size of the file, to show. */ + bytes: number; +} + +export type BakedAsset = PatternAsset | SongAsset; + +/** What a bake is worked out from: the same a game hears with `Play pattern` or `Play song`. */ +export interface BakeSource { + sequence: Sequence; + tempo: number; + /** In ticks: where it ends played once, and what it runs round looped. */ + length: number; + loop: LoopRange; + /** The tracks of a song, each on its bus. */ + buses: readonly BusLevel[]; +} + +/** Null where there is nothing to hear: a pattern without its instrument or notes, an empty song. */ +export function bakeSourceOf(project: Project, asset: BakedAsset): BakeSource | null { + if (asset.type === 'pattern') { + const sequence = patternSequence(asset); + if (sequence.notes.length === 0) return null; + const length = patternLength(asset); + return { sequence, tempo: patternTempo(asset.props), length, loop: { start: 0, end: length }, buses: [] }; + } + const sequence = songSequence(project, asset); + if (sequence.notes.length === 0 && !sequence.clips?.length) return null; + return { + sequence, + tempo: songTempo(asset.props), + length: songLength(asset), + loop: songLoop(asset), + buses: songBuses(project, asset), + }; +} + +/** Where the parts of a bake lie, in frames, for a source played at its tempo. */ +export function bakeSpans( + source: Pick, +): Omit { + const frames = (ticks: number) => Math.round(secondsOfTicks(ticks, source.tempo) * BAKE_RATE); + const length = frames(source.length); + const loopEnd = frames(source.loop.end); + return { length, loopEnd, head: Math.max(length, loopEnd), round: loopEnd - frames(source.loop.start) }; +} + +/** + * The playings a bake is made of: always one looped, as far as one more round past the loop end, and + * one played once only where it goes on past the loop end – up to there the two are the same. + */ +export function bakeRuns(spans: Pick): { + looped: number; + once: number | null; +} { + return { looped: spans.loopEnd + spans.round, once: spans.length > spans.loopEnd ? spans.length : null }; +} + +/** The file's samples, side by side: the head out of the playing once (or the looped one), then the round. */ +export function assembleBake( + spans: Pick, + looped: readonly Float32Array[], + once: readonly Float32Array[] | null, +): Float32Array[] { + return looped.map((side, c) => { + const out = new Float32Array(spans.head + spans.round); + const straight = once?.[c] ?? side; + out.set(straight.subarray(0, spans.head)); + out.set(side.subarray(spans.loopEnd, spans.loopEnd + spans.round), spans.head); + return out; + }); +} + +/** Raises the samples to full scale in place; gives back the loudest one before, 0 for silence. */ +export function normalizeBake(channels: readonly Float32Array[]): number { + let peak = 0; + for (const side of channels) for (const sample of side) peak = Math.max(peak, Math.abs(sample)); + if (peak > 0) for (const side of channels) for (let i = 0; i < side.length; i++) side[i]! /= peak; + return peak; +} + +/** + * How a bake is played (#340), in seconds into its file. Once: from 0 to `end`. Looped: from 0 up to + * `until`, then from `round.start` round and round to `round.end` – one source where the round follows + * at once, two where the head is longer than the loop end. + */ +export interface BakePlayback { + end: number | null; + until: number; + round: { start: number; end: number } | null; +} + +export function bakePlayback( + bake: Pick, + loop: boolean, +): BakePlayback { + const seconds = (frames: number) => frames / BAKE_RATE; + if (!loop || bake.round <= 0) + return { end: seconds(bake.length), until: seconds(bake.length), round: null }; + return { + end: null, + until: seconds(bake.loopEnd), + round: { start: seconds(bake.head), end: seconds(bake.head + bake.round) }, + }; +} + +/** + * What a bake is uploaded as: its owner's name behind a mark the sound picker leaves out by – in + * front, as the server cuts a long name at its end, and with no slash, which it takes for a folder. + */ +const BAKE_MARK = '[bake] '; + +export function bakeUploadName(assetName: string): string { + return `${BAKE_MARK}${assetName.replace(/[/\\]/g, '-')}.ogg`; +} + +export function isBakeUpload(name: string): boolean { + return name.startsWith(BAKE_MARK); +} + +/** Held to what a bake may be, else none: a file of the sounds, whole frames that fit together. */ +export function heldBake(raw: unknown): Bake | null { + if (typeof raw !== 'object' || raw === null) return null; + const b = raw as Record; + const whole = (value: unknown) => + typeof value === 'number' && Number.isInteger(value) && value >= 0 ? value : null; + const [length, loopEnd, head, round, bytes] = [b.length, b.loopEnd, b.head, b.round, b.bytes].map(whole); + const peak = typeof b.peak === 'number' && Number.isFinite(b.peak) && b.peak >= 0 ? b.peak : null; + if (typeof b.file !== 'string' || !isSoundFile(b.file) || typeof b.print !== 'string' || b.print === '') + return null; + if (length == null || loopEnd == null || head == null || round == null || bytes == null || peak === null) + return null; + if (head !== Math.max(length, loopEnd) || round > loopEnd) return null; + return { file: b.file, print: b.print, peak, length, loopEnd, head, round, bytes }; +} + +/** The bake of a pattern or song while it still holds, else null: something it is made of changed. */ +export function validBake(project: Project, asset: BakedAsset): Bake | null { + const bake = asset.bake; + return bake && bake.print === bakePrint(project, asset) ? bake : null; +} + +/** JSON of the plans the compilers keep: made once per plan, not once per note. */ +const planJson = new WeakMap(); + +function jsonOf(plan: object): string { + let json = planJson.get(plan); + if (json === undefined) planJson.set(plan, (json = JSON.stringify(plan))); + return json; +} + +/** + * What a bake is made from, as a short text: every note and clip with what plays it – the build of + * each instrument for each note it plays, its sound FX, the file, cut and level of each sound – the + * tempo, the length, the loop and the tracks. A change to any of them, also to an instrument or a + * sound the pattern only names, makes another. The name is not in it: renaming changes no sound. + * '' where there is nothing to bake. + */ +export function bakePrint(project: Project, asset: BakedAsset): string { + const source = bakeSourceOf(project, asset); + if (!source) return ''; + const { sequence, tempo } = source; + const sources = instrumentSourcesOf(project); + const instruments: Record }> = {}; + for (const { instrument, note } of sequence.notes) { + const found = findAssetOf(project, instrument, 'instrument'); + if (!found) continue; + const known = (instruments[instrument] ??= { + fx: jsonOf(fxVoiceOf(project, pickedFx('', found.props), tempo)?.plan ?? {}), + notes: {}, + }); + known.notes[note] ??= instrumentBuildFor(found, heldNote(note), sources).layers.map(jsonOf); + } + const clips = (sequence.clips ?? []).map((clip) => { + const ref = findSound(project, clip.sound); + return { + ...clip, + file: ref ? playedSoundOf(ref) : '', + level: ref ? volumeOf(resolveAssetProps(project, ref.asset)) : 0, + fx: jsonOf(clipFx(project, clip.fx, tempo).plan), + }; + }); + const buses = source.buses.map(({ fx, ...bus }) => ({ ...bus, fx: fx ? jsonOf(fx.plan) : '' })); + const text = JSON.stringify([ + BAKE_VERSION, + tempo, + source.length, + source.loop, + sequence.notes, + clips, + buses, + instruments, + ]); + return hashText(text); +} + +/** 53 bits of cyrb53, as base 36: no file depends on two prints never meeting, only a stale bake. */ +function hashText(text: string): string { + let h1 = 0xdeadbeef; + let h2 = 0x41c6ce57; + for (let i = 0; i < text.length; i++) { + const c = text.charCodeAt(i); + h1 = Math.imul(h1 ^ c, 2654435761); + h2 = Math.imul(h2 ^ c, 1597334677); + } + h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909); + h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909); + return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(36); +} diff --git a/src/core/music/ogg.ts b/src/core/music/ogg.ts new file mode 100644 index 00000000..e69a4063 --- /dev/null +++ b/src/core/music/ogg.ts @@ -0,0 +1,193 @@ +/** + * Opus in Ogg (RFC 7845), the file a bake is kept in (#340): the pages around the packets a WebCodecs + * `AudioEncoder` hands out, and the way back. The head says how many samples the encoder ran ahead + * (`preSkip`) and the last page how many there are, so a decoder gives back exactly what went in, + * to the sample. + */ + +/** Opus always counts at 48 kHz, whatever the rate of what went in. */ +export const OPUS_RATE = 48000; + +/** One packet as the encoder gave it, and how many samples per channel it stands for. */ +export interface OpusPacket { + data: Uint8Array; + samples: number; +} + +export interface OpusStream { + channels: number; + preSkip: number; + /** Samples per channel of what went in, without the pre-skip. */ + frames: number; + packets: readonly OpusPacket[]; +} + +/** What a page holds at most: 255 lacing values, and so at most a packet of about 64 kB. */ +const MAX_SEGMENTS = 255; +const SERIAL = 0x6d616b65; +const VENDOR = 'game-maker'; + +const CRC_TABLE = (() => { + const table = new Uint32Array(256); + for (let i = 0; i < 256; i++) { + let r = i << 24; + for (let b = 0; b < 8; b++) r = r & 0x80000000 ? (r << 1) ^ 0x04c11db7 : r << 1; + table[i] = r >>> 0; + } + return table; +})(); + +/** The checksum of an Ogg page: CRC-32 with the polynomial 0x04c11db7, not reflected, from 0. */ +export function oggCrc(bytes: Uint8Array): number { + let crc = 0; + for (const byte of bytes) crc = ((crc << 8) ^ CRC_TABLE[((crc >>> 24) ^ byte) & 0xff]!) >>> 0; + return crc; +} + +/** The identification header: version 1, mapping family 0 (mono or stereo), no output gain. */ +export function opusHead(channels: number, preSkip: number): Uint8Array { + const head = new Uint8Array(19); + const view = new DataView(head.buffer); + head.set(ascii('OpusHead')); + head[8] = 1; + head[9] = channels; + view.setUint16(10, preSkip, true); + view.setUint32(12, OPUS_RATE, true); + return head; +} + +/** The pre-skip an identification header names, as an encoder hands one out; null for anything else. */ +export function preSkipOf(head: Uint8Array): number | null { + if (head.length < 19 || text(head.subarray(0, 8)) !== 'OpusHead') return null; + return new DataView(head.buffer, head.byteOffset, head.byteLength).getUint16(10, true); +} + +function opusTags(): Uint8Array { + const vendor = ascii(VENDOR); + const tags = new Uint8Array(8 + 4 + vendor.length + 4); + const view = new DataView(tags.buffer); + tags.set(ascii('OpusTags')); + view.setUint32(8, vendor.length, true); + tags.set(vendor, 12); + return tags; +} + +/** + * The whole file: the head and the tags on a page each, then the packets, as many on a page as fit. + * A page's granule position counts every sample finished on it, the pre-skip among them; the last + * one stops at the end of what went in, which cuts off what the encoder padded the last packet with. + */ +export function oggOpus(stream: OpusStream): Uint8Array { + const pages: Uint8Array[] = []; + let sequence = 0; + const page = (packets: readonly Uint8Array[], granule: number, flags: number) => + pages.push(oggPage(packets, granule, flags, sequence++)); + page([opusHead(stream.channels, stream.preSkip)], 0, 0x02); + page([opusTags()], 0, 0); + const end = stream.preSkip + stream.frames; + let granule = 0; + let held: Uint8Array[] = []; + let segments = 0; + stream.packets.forEach((packet, i) => { + const needs = Math.floor(packet.data.length / 255) + 1; + if (segments + needs > MAX_SEGMENTS) { + page(held, Math.min(granule, end), 0); + held = []; + segments = 0; + } + held.push(packet.data); + segments += needs; + granule += packet.samples; + if (i === stream.packets.length - 1) page(held, end, 0x04); + }); + if (stream.packets.length === 0) page([], end, 0x04); + return concat(pages); +} + +function oggPage( + packets: readonly Uint8Array[], + granule: number, + flags: number, + sequence: number, +): Uint8Array { + const lacing: number[] = []; + for (const packet of packets) { + for (let left = packet.length; left >= 255; left -= 255) lacing.push(255); + lacing.push(packet.length % 255); + } + const body = concat(packets); + const page = new Uint8Array(27 + lacing.length + body.length); + const view = new DataView(page.buffer); + page.set(ascii('OggS')); + page[5] = flags; + view.setUint32(6, granule % 2 ** 32, true); + view.setUint32(10, Math.floor(granule / 2 ** 32), true); + view.setUint32(14, SERIAL, true); + view.setUint32(18, sequence, true); + page[26] = lacing.length; + page.set(lacing, 27); + page.set(body, 27 + lacing.length); + view.setUint32(22, oggCrc(page), true); + return page; +} + +/** What `readOggOpus` finds: the head, the last granule position and every packet after the tags. */ +export interface ReadOpus { + channels: number; + preSkip: number; + granule: number; + packets: Uint8Array[]; + pages: number; +} + +/** Reads a file `oggOpus` wrote, checking every page; throws at the first that is not right. */ +export function readOggOpus(bytes: Uint8Array): ReadOpus { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const all: Uint8Array[] = []; + let open: Uint8Array[] = []; + let granule = 0; + let pages = 0; + for (let at = 0; at < bytes.length; pages++) { + if (text(bytes.subarray(at, at + 4)) !== 'OggS') throw new Error(`no page at byte ${at}`); + const count = bytes[at + 26]!; + const lacing = bytes.subarray(at + 27, at + 27 + count); + const size = 27 + count + lacing.reduce((sum, value) => sum + value, 0); + const page = bytes.slice(at, at + size); + const crc = view.getUint32(at + 22, true); + page.fill(0, 22, 26); + if (oggCrc(page) !== crc) throw new Error(`page ${pages} has a wrong checksum`); + granule = view.getUint32(at + 6, true) + view.getUint32(at + 10, true) * 2 ** 32; + let from = at + 27 + count; + for (const value of lacing) { + open.push(bytes.subarray(from, from + value)); + from += value; + if (value < 255) { + all.push(concat(open)); + open = []; + } + } + at += size; + } + const head = all[0]; + const preSkip = head ? preSkipOf(head) : null; + if (!head || preSkip === null) throw new Error('no OpusHead'); + return { channels: head[9]!, preSkip, granule, packets: all.slice(2), pages }; +} + +function ascii(value: string): Uint8Array { + return Uint8Array.from(value, (char) => char.charCodeAt(0)); +} + +function text(bytes: Uint8Array): string { + return String.fromCharCode(...bytes); +} + +function concat(parts: readonly Uint8Array[]): Uint8Array { + const out = new Uint8Array(parts.reduce((sum, part) => sum + part.length, 0)); + let at = 0; + for (const part of parts) { + out.set(part, at); + at += part.length; + } + return out; +} diff --git a/src/core/music/playbacks.ts b/src/core/music/playbacks.ts index 51ea19c4..d1202146 100644 --- a/src/core/music/playbacks.ts +++ b/src/core/music/playbacks.ts @@ -1,5 +1,6 @@ import type { SendTap } from '../graph/Graph'; import type { FxPlan, FxVoice } from '../soundfx/nodes'; +import type { BakePlayback } from './bake'; import type { MixSend } from './mixer'; import { LEAD, type ScheduledClip, type ScheduledNote, Scheduler, type SchedulerHost } from './Scheduler'; import type { LoopRange, Sequence } from './sequence'; @@ -53,6 +54,26 @@ export interface SequencePlay { /** Starts one note as the player was asked to, and gives back the number of its voice. */ strike(note: ScheduledNote): number; strikeClip?(clip: ScheduledClip): number; + /** Its bake (#340), played in place of its notes and clips where the host can. */ + bake?: BakePlay; +} + +/** A bake as it is played: its file, the level it is heard at, and which stretches of it. */ +export interface BakePlay { + file: string; + gain: number; + playback: BakePlayback; +} + +/** What a playing is driven by: a scheduler of its notes, or a bake. */ +export interface Runner { + readonly playing: boolean; + /** The second of the audio clock it is heard from; null before it starts. */ + readonly startedAt: number | null; + /** Played to its end, and nothing of it is heard any more. */ + readonly done: boolean; + start(from?: number): void; + stop(letRing?: boolean): void; } export interface PlaybackHost extends Omit { @@ -66,6 +87,8 @@ export interface PlaybackHost extends Omit void): void; /** A playing is gone: stopped, or played to its end. */ closed?(id: number): void; + /** Starts the bake of that playing to be heard from `at`, as one voice; 0 where nothing plays. */ + playBaked?(id: number, bake: BakePlay, at: number): number; } export interface Playback { @@ -73,12 +96,14 @@ export interface Playback { /** What stands for the pattern or song, as a file stands for a sound: `pattern:`, `song:`. */ name: string; owner: string; - scheduler: Scheduler; + runner: Runner; /** The last notes and clips planned, newest last, for a check to hold against the clock. */ planned: ScheduledNote[]; clips: ScheduledClip[]; /** The second of the audio clock a fade out ends it at; null while it is not fading. */ fading: number | null; + /** Played from its bake (#340) rather than its notes. */ + baked: boolean; } /** Whether a playing goes by `name`: its own, every one of a kind for `kind:`, every one for `''`. */ @@ -106,23 +131,38 @@ export class Playbacks { if (list.length > KEPT) list.shift(); }; const strikeClip = play.strikeClip; - const scheduler = new Scheduler( - { - ...this.host, - ahead: () => this.host.ahead?.(id) ?? 0, - play: (note) => (keep(planned, note), play.strike(note)), - ...(strikeClip && { playClip: (clip: ScheduledClip) => (keep(clips, clip), strikeClip(clip)) }), - }, - () => play.sequence, - { tempo: play.tempo, loop: play.loop, end: play.length }, - ); - const playback: Playback = { id, name, owner, scheduler, planned, clips, fading: null }; + const ahead = () => this.host.ahead?.(id) ?? 0; + const { bake } = play; + const playBaked = this.host.playBaked?.bind(this.host); + const runner: Runner = + bake && playBaked + ? new BakedRun(this.host, ahead, (at) => playBaked(id, bake, at), bake.playback.end !== null) + : new Scheduler( + { + ...this.host, + ahead, + play: (note) => (keep(planned, note), play.strike(note)), + ...(strikeClip && { playClip: (clip: ScheduledClip) => (keep(clips, clip), strikeClip(clip)) }), + }, + () => play.sequence, + { tempo: play.tempo, loop: play.loop, end: play.length }, + ); + const playback: Playback = { + id, + name, + owner, + runner, + planned, + clips, + fading: null, + baked: runner instanceof BakedRun, + }; this.list.push(playback); void this.host.prepare(play.needs, id).then(() => { if (!this.list.includes(playback) || playback.fading !== null) return; - scheduler.start(0); + runner.start(0); // From the first note heard, which is later where the buses are (#346). - const at = scheduler.startedAt ?? this.host.now() + LEAD; + const at = runner.startedAt ?? this.host.now() + LEAD; this.host.level(id, play.volume ?? 1, at, Math.max(0, play.fadeIn ?? 0)); }); } @@ -148,7 +188,7 @@ export class Playbacks { /** What is playing, oldest first (test hook). */ current(): readonly Readonly[] { - const gone = this.list.filter((playback) => playback.scheduler.done); + const gone = this.list.filter((playback) => playback.runner.done); if (gone.length > 0) this.close(gone, false); return this.list; } @@ -173,8 +213,49 @@ export class Playbacks { if (going.length === 0) return; this.list = this.list.filter((playback) => !going.includes(playback)); for (const playback of going) { - playback.scheduler.stop(letRing); + playback.runner.stop(letRing); this.host.closed?.(playback.id); } } } + +/** + * A bake played in place of the notes (#340): one voice from the moment the first note would be + * heard, ended as the notes would be. One that ends is done once its voice is. + */ +class BakedRun implements Runner { + private voice = 0; + private anchor: number | null = null; + + constructor( + private readonly host: Pick, + private readonly ahead: () => number, + private readonly play: (at: number) => number, + private readonly ends: boolean, + ) {} + + get playing(): boolean { + return this.anchor !== null; + } + + get startedAt(): number | null { + return this.anchor; + } + + get done(): boolean { + return this.anchor !== null && this.ends && !this.host.playing(this.voice); + } + + start(): void { + this.stop(); + const ahead = this.ahead(); + this.anchor = this.host.now() + LEAD + (Number.isFinite(ahead) && ahead > 0 ? ahead : 0); + this.voice = this.play(this.anchor); + } + + stop(letRing = false): void { + if (this.voice !== 0) this.host.stop([this.voice], letRing); + this.voice = 0; + this.anchor = null; + } +} diff --git a/src/core/store/serialize.ts b/src/core/store/serialize.ts index 93213eae..c8508779 100644 --- a/src/core/store/serialize.ts +++ b/src/core/store/serialize.ts @@ -21,6 +21,7 @@ import { isParticleNodeDef } from '../particles/nodes'; import { isShaderNodeDef } from '../shader/nodes'; import { isSoundFxNodeDef } from '../soundfx/nodes'; import { INSTRUMENT_SAMPLE, isInstrumentNodeDef, withSamplePlay } from '../instrument/nodes'; +import { heldBake } from '../music/bake'; import { heldPatternNotes } from '../music/pattern'; import { isEmbeddable } from '../music/embedded'; import { heldSongItems, heldSongTracks } from '../music/song'; @@ -284,9 +285,19 @@ export function deserializeProject(raw: unknown, fallbackId = ''): Project { assets.push({ id, type, name, props, colors: normalizePalette(a.colors) }); } else if (type === 'pattern') { const loop = readLoop(a.loop); - assets.push({ id, type, name, props, notes: heldPatternNotes(a.notes), ...(loop && { loop }) }); + const bake = heldBake(a.bake); + assets.push({ + id, + type, + name, + props, + notes: heldPatternNotes(a.notes), + ...(loop && { loop }), + ...(bake && { bake }), + }); } else if (type === 'song') { const loop = readLoop(a.loop); + const bake = heldBake(a.bake); const tracks = heldSongTracks(a.tracks); assets.push({ id, @@ -297,6 +308,7 @@ export function deserializeProject(raw: unknown, fallbackId = ''): Project { items: heldSongItems(a.items, tracks), ...(loop && { loop }), graph: heldSongGraph(normalizeGraph(a.graph, isSongRoutingDef), tracks), + ...(bake && { bake }), }); } else if (type === 'curve') { assets.push({ id, type, name, props, points: sanitizedCurve(normalizePoints(a.points)) }); diff --git a/src/editor/Editor.ts b/src/editor/Editor.ts index 075bdb34..538ce83e 100644 --- a/src/editor/Editor.ts +++ b/src/editor/Editor.ts @@ -352,7 +352,7 @@ export class Editor implements PlayHost { this.soundEditor = new SoundEditor(state); this.soundFxEditor = new SoundFxEditor(state); this.instrumentEditor = new InstrumentEditor(state); - this.patternEditor = new PatternEditor(state); + this.patternEditor = new PatternEditor(state, sounds); this.songEditor = new SongEditor(state, sounds); this.textureEditor = new TextureEditor(state); this.vectorArtEditor = new VectorArtEditor(state); diff --git a/src/editor/debug/bakeHooks.ts b/src/editor/debug/bakeHooks.ts new file mode 100644 index 00000000..a021cf81 --- /dev/null +++ b/src/editor/debug/bakeHooks.ts @@ -0,0 +1,65 @@ +import { BAKE_RATE, type BakedAsset, bakePlayback, bakeSourceOf, validBake } from '../../core/music/bake'; +import { findAsset } from '../../core/model/Project'; +import { renderRun } from '../../viewport/bakeRender'; +import { centreOf, type HookContext } from './hookHelpers'; + +/** Bakes of patterns and songs (#340). */ +export function bakeHooks({ state, editor }: HookContext) { + const asset = (id: string): BakedAsset | null => { + const found = findAsset(state.project, id); + return found?.type === 'pattern' || found?.type === 'song' ? found : null; + }; + const button = (id: string) => + asset(id)?.type === 'song' ? editor.songEditor.bakeButton : editor.patternEditor.bakeButton; + return { + bake: { + /** Bakes it as Bake does in its tab; done once it is baked, failed or cancelled. */ + run: (id: string) => button(id).bake(asset(id)), + cancel: (id: string) => button(id).cancel(), + /** What it keeps of its bake, whether that still holds, and the button of the tab showing it. */ + info: (id: string) => { + const found = asset(id); + const shown = button(id); + return { + bake: found?.bake ?? null, + valid: found !== null && validBake(state.project, found) !== null, + status: shown.status(found), + button: { + ...centreOf(shown), + toggled: shown.toggled, + enabled: shown.enabled, + tooltip: shown.tooltipText(), + }, + }; + }, + /** + * Plays `seconds` of it offline from its notes and from its bake, looped or once, and holds the + * two together: the level of every `window` seconds of each in dBFS, -Infinity for silence. + */ + compare: async (id: string, loop: boolean, seconds: number, window: number) => { + const found = asset(id); + const bake = found && validBake(state.project, found); + const source = found && bakeSourceOf(state.project, found); + if (!bake || !source) return null; + const frames = Math.round(seconds * BAKE_RATE); + const watch = { progress: () => {}, cancelled: () => false }; + const played = { file: bake.file, gain: bake.peak, playback: bakePlayback(bake, loop) }; + const live = await renderRun(state.project, source, loop, frames, null, watch); + const baked = await renderRun(state.project, source, loop, frames, played, watch); + return { live: levels(live.channels, window), baked: levels(baked.channels, window) }; + }, + }, + }; +} + +function levels(channels: readonly Float32Array[], window: number): number[] { + const span = Math.round(window * BAKE_RATE); + const length = channels[0]?.length ?? 0; + const out: number[] = []; + for (let from = 0; from + span <= length; from += span) { + let sum = 0; + for (const side of channels) for (let i = from; i < from + span; i++) sum += side[i]! * side[i]!; + out.push(10 * Math.log10(sum / (span * channels.length))); + } + return out; +} diff --git a/src/editor/debugHook.ts b/src/editor/debugHook.ts index d0dbc935..84200395 100644 --- a/src/editor/debugHook.ts +++ b/src/editor/debugHook.ts @@ -7,6 +7,7 @@ import { Music } from '../viewport/music'; import { bothAtCenter, envelopeDriver, followCurve, releaseEnvelope, stageNodes } from '../viewport/fxNodes'; import { soundLibrary } from '../viewport/SoundLibrary'; import { saturationAutoGain } from '../core/soundfx/saturation'; +import { bakeHooks } from './debug/bakeHooks'; import { curveHooks } from './debug/curveHooks'; import { soundHooks } from './debug/soundHooks'; import { patternHooks } from './debug/patternHooks'; @@ -47,6 +48,7 @@ export function installDebugHook(app: Application, state: EditorState, editor: E ...songHooks(context), ...routingHooks(context), ...soundHooks(context), + ...bakeHooks(context), ...renderHooks(context), /** * Voices started, stopped and in flight. CI runs the browser with `--mute-audio`, so nothing diff --git a/src/editor/music/BakeButton.ts b/src/editor/music/BakeButton.ts new file mode 100644 index 00000000..7c88f32a --- /dev/null +++ b/src/editor/music/BakeButton.ts @@ -0,0 +1,178 @@ +import { + BAKE_RATE, + type BakedAsset, + bakePrint, + bakeSourceOf, + bakeUploadName, + validBake, +} from '../../core/music/bake'; +import { findAsset } from '../../core/model/Project'; +import type { SoundStore } from '../../core/store/SoundStore'; +import { Button, formatBytes, Menu, Theme } from '../../ui'; +import { BakeCancelled, makeBake } from '../../viewport/bakeRender'; +import { HEADER_ICON_SVG } from '../panels/headerIcons'; +import type { EditorState } from '../state'; +import { clockText } from './timeView'; + +/** Where a bake stands for the asset shown. */ +export type BakeStatus = 'none' | 'baking' | 'baked' | 'outdated'; + +/** + * Bake in the header of the Pattern and the Song tab (#340), an icon – the Song tab's header has no + * room for a caption: works out once what the one shown plays, puts it up as a file and keeps that on + * the asset, and a game plays it from there. It is lit while the bake holds. While it runs, the + * toolbar says how far it has got and a click cancels it; once baked, a click offers to bake it again + * or to remove the bake. The tab itself always plays live. + */ +export class BakeButton extends Button { + private job: { asset: BakedAsset; share: number; cancelled: boolean; done: Promise } | null = null; + + /** `refreshed` refreshes the tab: a bake starting or ending changes what its info says. */ + constructor( + private readonly state: EditorState, + private readonly sounds: SoundStore, + private readonly shown: () => BakedAsset | null, + private readonly refreshed: () => void, + ) { + super('', { width: Theme.controlHeight, icon: HEADER_ICON_SVG.bake, toggle: true }); + this.tooltip = () => this.tip(); + this.onClick = () => this.clicked(); + } + + status(asset = this.shown()): BakeStatus { + if (this.job) return 'baking'; + if (!asset?.bake) return 'none'; + return validBake(this.state.project, asset) ? 'baked' : 'outdated'; + } + + /** What the info of the tab adds: '' where there is no bake. */ + statusText(asset: BakedAsset): string { + const status = this.status(asset); + return status === 'baked' ? 'baked' : status === 'outdated' ? 'bake outdated' : ''; + } + + /** Lights it while the bake of the asset shown holds; called whenever the project changes. */ + sync(): void { + const asset = this.shown(); + this.setToggled(this.status(asset) === 'baked'); + const bakeable = asset !== null && bakeSourceOf(this.state.project, asset) !== null; + this.setEnabled( + this.job !== null || (this.state.mode !== 'play' && (bakeable || asset?.bake !== undefined)), + ); + } + + /** Bakes the asset shown; done once it is baked, failed or cancelled (test hook). */ + bake(asset = this.shown()): Promise { + if (this.job) return this.job.done; + if (!asset || this.state.mode === 'play') return Promise.resolve(); + const job = { asset, share: 0, cancelled: false, done: Promise.resolve() }; + this.job = job; + job.done = this.run(job).finally(() => { + this.job = null; + this.refreshed(); + }); + this.refreshed(); + return job.done; + } + + cancel(): void { + if (this.job) this.job.cancelled = true; + } + + private clicked(): void { + if (this.job) return this.cancel(); + const asset = this.shown(); + if (!asset) return; + if (!asset.bake) return void this.bake(asset); + new Menu([ + { label: 'Bake again', onSelect: () => void this.bake(asset) }, + { label: 'Remove bake', onSelect: () => this.remove(asset) }, + ]).openBelow(this); + } + + private tip(): string { + const asset = this.shown(); + switch (this.status(asset)) { + case 'baking': + return `Baking ${this.job?.asset.name ?? ''}: ${Math.round((this.job?.share ?? 0) * 100)}% – click to cancel`; + case 'baked': { + const bake = asset!.bake!; + return `Baked: ${clockText(bake.length / BAKE_RATE)}, ${formatBytes(bake.bytes)}. A game plays the bake, this tab plays live`; + } + case 'outdated': + return 'Changed since it was baked, so a game plays it live. Bake it again, or remove the bake'; + default: + return 'Bake: work out once what it plays, so a game plays that instead of every note'; + } + } + + private async run(job: { asset: BakedAsset; share: number; cancelled: boolean }): Promise { + const { asset } = job; + try { + const made = await makeBake(this.state.project, asset, { + progress: (share) => { + const before = Math.round(job.share * 100); + job.share = share; + const now = Math.round(share * 100); + if (now !== before) this.state.setNotice(`Baking ${asset.name}: ${now}%`, 'quiet'); + }, + cancelled: () => job.cancelled, + }); + const { data, ...rest } = made; + if (!this.unchanged(asset, made.print)) return this.changed(asset); + const uploaded = await this.sounds.upload(data.slice().buffer, 'audio/ogg', bakeUploadName(asset.name)); + // Nothing names the file yet, so it can go at once. + if (!this.unchanged(asset, made.print)) { + void this.sounds.delete(uploaded.file).catch(() => {}); + return this.changed(asset); + } + const old = this.current(asset.id)?.bake?.file; + this.state.setBake(asset.id, { file: uploaded.file, bytes: uploaded.bytes, ...rest }); + if (old && old !== uploaded.file) this.dropWhenSaved(old); + this.state.setNotice( + `Baked ${asset.name}: ${clockText(rest.length / BAKE_RATE)}, ${formatBytes(uploaded.bytes)}`, + ); + } catch (error: unknown) { + if (error instanceof BakeCancelled) this.state.setNotice(`Baking ${asset.name} was cancelled`); + else + this.state.setNotice( + `Baking ${asset.name} failed: ${error instanceof Error ? error.message : String(error)}`, + 'error', + ); + } + } + + private remove(asset: BakedAsset): void { + const old = asset.bake?.file; + this.state.setBake(asset.id, null); + if (old) this.dropWhenSaved(old); + } + + private current(id: string): BakedAsset | null { + const asset = findAsset(this.state.project, id); + return asset?.type === 'pattern' || asset?.type === 'song' ? asset : null; + } + + /** Whether it still sounds as it did when the bake was started. */ + private unchanged(asset: BakedAsset, print: string): boolean { + const now = this.current(asset.id); + return now !== null && bakePrint(this.state.project, now) === print; + } + + private changed(asset: BakedAsset): void { + this.state.setNotice(`${asset.name} changed while it was baked – bake it again`, 'error'); + } + + /** + * Deletes the file of a bake that has gone, once the project no longer names it on the server: the + * server keeps a file any saved project or snapshot names (409), and so does this. + */ + private dropWhenSaved(file: string): void { + const off = this.state.on('save', () => { + const status = this.state.saveStatus; + if (status === 'dirty' || status === 'saving') return; + off(); + if (status === 'saved') void this.sounds.delete(file).catch(() => {}); + }); + } +} diff --git a/src/editor/music/timeView.ts b/src/editor/music/timeView.ts index 4b77b1f6..9343bb45 100644 --- a/src/editor/music/timeView.ts +++ b/src/editor/music/timeView.ts @@ -132,3 +132,9 @@ export function barLabelStep(view: TimeView, meter: Meter): number { while (n * barPx < BAR_LABEL_MIN_PX && n < 1024) n *= 2; return n; } + +/** "1:05" */ +export function clockText(seconds: number): string { + const whole = Math.round(seconds); + return `${Math.floor(whole / 60)}:${String(whole % 60).padStart(2, '0')}`; +} diff --git a/src/editor/panels/headerIcons.ts b/src/editor/panels/headerIcons.ts index 65aeb9ea..3ec396e9 100644 --- a/src/editor/panels/headerIcons.ts +++ b/src/editor/panels/headerIcons.ts @@ -1,5 +1,6 @@ /** Icons of the tab headers' switches (#194): a picture instead of a caption. */ +import bake from './icons/bake.svg?raw'; import magnet from './icons/magnet.svg?raw'; import reverse from './icons/reverse.svg?raw'; -export const HEADER_ICON_SVG = { snap: magnet, reverse }; +export const HEADER_ICON_SVG = { snap: magnet, reverse, bake }; diff --git a/src/editor/panels/icons/bake.svg b/src/editor/panels/icons/bake.svg new file mode 100644 index 00000000..83872cbe --- /dev/null +++ b/src/editor/panels/icons/bake.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/editor/pattern/PatternEditor.ts b/src/editor/pattern/PatternEditor.ts index e9472502..36f0cab8 100644 --- a/src/editor/pattern/PatternEditor.ts +++ b/src/editor/pattern/PatternEditor.ts @@ -26,7 +26,9 @@ import { Button, Dropdown, Label, Menu, type Widget } from '../../ui'; import { Music } from '../../viewport/music'; import { MASTER_MIX } from '../../core/music/mixer'; import { soundLibrary } from '../../viewport/SoundLibrary'; +import type { SoundStore } from '../../core/store/SoundStore'; import { AssetTabEditor } from '../AssetTabEditor'; +import { BakeButton } from '../music/BakeButton'; import { claimTransport, releaseTransport } from '../music/transport'; import { TOOLBAR_ICON_SVG } from '../panels/toolbarIcons'; import type { EditorState } from '../state'; @@ -42,7 +44,8 @@ const PREVIEW_OWNER = 'pattern-preview'; /** * The Pattern tab (#333): the piano roll, and in the header Play – the Space bar does the same – * and the grid notes are drawn on. The pattern plays round its loop through the scheduler (#332); - * an edit is heard from its next window on, without starting over. + * an edit is heard from its next window on, without starting over. Bake (#340) is for a game: the + * tab always plays the notes. */ export class PatternEditor extends AssetTabEditor { /** Public for the check scripts. */ @@ -54,6 +57,7 @@ export class PatternEditor extends AssetTabEditor { readonly ghostsButton: Button; /** Where the ghost notes come from: song, track and bar. */ readonly ghostSource = new Label('', { style: 'muted' }); + readonly bakeButton: BakeButton; /** Heard through the sound FX (#352), then straight into the master: a preview leaves the routing out (#368). */ readonly music = new Music( () => this.state.project, @@ -69,8 +73,14 @@ export class PatternEditor extends AssetTabEditor { private told = ''; private readonly halt = () => this.stop(); - constructor(state: EditorState) { + constructor(state: EditorState, sounds: SoundStore) { super(state, { infoWidth: INFO_WIDTH }); + this.bakeButton = new BakeButton( + state, + sounds, + () => this.asset(), + () => this.refresh(), + ); this.playButton = new Button('Play', { icon: TOOLBAR_ICON_SVG.play, minWidth: 72, @@ -111,6 +121,7 @@ export class PatternEditor extends AssetTabEditor { this.gridButton, this.auditionButton, this.ghostsButton, + this.bakeButton, this.ghostSource, ]); state.on('load', () => this.stop()); @@ -238,7 +249,9 @@ export class PatternEditor extends AssetTabEditor { const count = asset.notes.length; const notes = `${count} ${count === 1 ? 'note' : 'notes'}`; const instrument = findAssetOf(this.state.project, patternInstrument(asset.props), 'instrument'); - return instrument ? `${notes} · ${instrument.name}` : `${notes} · no instrument`; + const text = instrument ? `${notes} · ${instrument.name}` : `${notes} · no instrument`; + const bake = this.bakeButton.statusText(asset); + return bake ? `${text} · ${bake}` : text; } protected override syncExtras(asset: PatternAsset | null): void { @@ -257,6 +270,7 @@ export class PatternEditor extends AssetTabEditor { this.canvas.setInstrument(new Set(sounding), namedNotes(instrument.graph)); } else this.canvas.setInstrument(new Set(), new Map()); this.syncGhosts(asset); + this.bakeButton.sync(); this.follow(asset); } diff --git a/src/editor/song/SongEditor.ts b/src/editor/song/SongEditor.ts index 56fb84a3..190de2d0 100644 --- a/src/editor/song/SongEditor.ts +++ b/src/editor/song/SongEditor.ts @@ -41,6 +41,8 @@ import { activateAsset } from '../actions'; import { AssetTabEditor } from '../AssetTabEditor'; import { GraphCanvas } from '../graph/GraphCanvas'; import { SnapToggle } from '../graph/SnapToggle'; +import { BakeButton } from '../music/BakeButton'; +import { clockText } from '../music/timeView'; import { claimTransport, releaseTransport } from '../music/transport'; import { TOOLBAR_ICON_SVG } from '../panels/toolbarIcons'; import type { EditorState } from '../state'; @@ -89,6 +91,7 @@ export class SongEditor extends AssetTabEditor { /** Public for the check scripts. */ readonly timelineButton: Button; readonly routingButton: Button; + readonly bakeButton: BakeButton; readonly routing: GraphCanvas; readonly snap = new SnapToggle(() => this.routing); private shown: SongView = 'timeline'; @@ -141,6 +144,12 @@ export class SongEditor extends AssetTabEditor { onClick: () => this.showView('routing'), }); this.routing = new GraphCanvas(state, 'song'); + this.bakeButton = new BakeButton( + state, + sounds, + () => this.asset(), + () => this.refresh(), + ); this.inputButton.onClick = () => void this.openInputMenu(); this.gridButton.onClick = () => this.openGridMenu(); const timeline = new SongTimeline(state); @@ -177,6 +186,7 @@ export class SongEditor extends AssetTabEditor { this.meter, new Label('Grid', { style: 'muted' }), this.gridButton, + this.bakeButton, ]); state.on('load', () => this.stop({ discard: true })); void this.listInputs(); @@ -567,7 +577,9 @@ export class SongEditor extends AssetTabEditor { protected override infoText(asset: SongAsset): string { const tracks = asset.tracks.length; const items = asset.items.length; - return `${tracks} ${tracks === 1 ? 'track' : 'tracks'} · ${items} ${items === 1 ? 'item' : 'items'}`; + const text = `${tracks} ${tracks === 1 ? 'track' : 'tracks'} · ${items} ${items === 1 ? 'item' : 'items'}`; + const bake = this.bakeButton.statusText(asset); + return bake ? `${text} · ${bake}` : text; } protected override syncExtras(asset: SongAsset | null): void { @@ -577,16 +589,11 @@ export class SongEditor extends AssetTabEditor { this.syncPlayButton(); const grid = textOf(asset?.props[SONG_GRID_KEY] ?? DEFAULT_SONG_GRID); this.gridButton.setText(SONG_GRID_OPTIONS.find((o) => o.value === grid)?.label ?? DEFAULT_SONG_GRID); + this.bakeButton.sync(); this.follow(asset); } } -/** "1:05" */ -function clockText(seconds: number): string { - const whole = Math.round(seconds); - return `${Math.floor(whole / 60)}:${String(whole % 60).padStart(2, '0')}`; -} - /** Hands the take to the browser as a file, for when the server would not take it. */ function downloadWav(wav: Uint8Array, name: string): void { const url = URL.createObjectURL(new Blob([wav], { type: 'audio/wav' })); diff --git a/src/editor/sound/SoundPicker.ts b/src/editor/sound/SoundPicker.ts index ca95465a..d193d750 100644 --- a/src/editor/sound/SoundPicker.ts +++ b/src/editor/sound/SoundPicker.ts @@ -1,4 +1,5 @@ import { Container, type FederatedPointerEvent, Graphics, type Text } from 'pixi.js'; +import { isBakeUpload } from '../../core/music/bake'; import type { StoredSound } from '../../core/store/SoundStore'; import { drawSvg, formatBytes, textStyle, Theme, truncate, crispText } from '../../ui'; import { soundLibrary } from '../../viewport/SoundLibrary'; @@ -118,8 +119,10 @@ class SoundPicker extends FilePicker { soundLibrary.stopAll(); } + /** A bake (#340) is in the sounds, but no sound to pick. */ protected async fetchItems(): Promise { - return withAssetNames(await this.deps.sounds.list(), this.deps.state.project, 'sound'); + const sounds = (await this.deps.sounds.list()).filter((sound) => !isBakeUpload(sound.name)); + return withAssetNames(sounds, this.deps.state.project, 'sound'); } protected contentHeight(count: number): { unit: number; total: number } { diff --git a/src/editor/state.ts b/src/editor/state.ts index be093429..ff1d4741 100644 --- a/src/editor/state.ts +++ b/src/editor/state.ts @@ -1,5 +1,6 @@ import type { KeyClipboard } from '../core/commands/animationCommands'; import type { PatternPlace } from '../core/music/ghosts'; +import type { Bake } from '../core/music/bake'; import type { EditorClipboard } from './clipboard'; import type { ObjectPick, PickEnd } from './objectPick'; import { findKey, findTrack, hasKey, type KeyRef, sameKey } from '../core/model/animation'; @@ -427,6 +428,21 @@ export class EditorState { this.emit('project'); } + /** + * The bake of a pattern or song (#340), or none: saved, and like the game scale no undo step – an + * undo back to what was baked finds it again. + */ + setBake(assetId: string, bake: Bake | null): void { + for (const project of [this.project, this.edit?.project]) { + const asset = project && findAsset(project, assetId); + if (asset?.type !== 'pattern' && asset?.type !== 'song') continue; + if (bake) asset.bake = bake; + else delete asset.bake; + } + this.setSaveStatus('dirty'); + this.emit('project'); + } + /** * Runs a command through the undo stack. Returns false (and sets a notice) when invalid. * In play mode the command changes the runtime copy only (see `enterPlay`). diff --git a/src/runtime/systems/behaviorContext.ts b/src/runtime/systems/behaviorContext.ts index fb9d7605..adf45b2a 100644 --- a/src/runtime/systems/behaviorContext.ts +++ b/src/runtime/systems/behaviorContext.ts @@ -20,6 +20,7 @@ import { loopsOf, volumeOf } from '../../core/properties/sounds'; import { instrumentBuildFor, instrumentSourcesOf, throughScopedFx } from '../../core/instrument/compile'; import { VELOCITY_PARAM } from '../../core/instrument/nodes'; import { heldNote } from '../../core/instrument/notes'; +import { bakePlayback, validBake } from '../../core/music/bake'; import { clipFx, fxVoiceOf, sequenceNeeds } from '../../core/music/needs'; import { patternInstrument, patternLength, patternSequence, patternTempo } from '../../core/music/pattern'; import type { ScheduledClip, ScheduledNote } from '../../core/music/Scheduler'; @@ -366,7 +367,8 @@ function heardOn(route: MixRoute): Pick { /** * A pattern played by a game: its notes through its instrument at its tempo, the whole of it once or * round and round – the loop set on its ruler is for editing. Its notes take the volume and the owner - * of the playing. Returns the number of the playing, 0 where there is no pattern or nothing to hear. + * of the playing; its bake (#340) takes the volume where it still holds, and is played instead. + * Returns the number of the playing, 0 where there is no pattern or nothing to hear. */ function playPattern( { project, audio }: Pick, @@ -387,6 +389,7 @@ function playPattern( const length = patternLength(asset); const tempo = patternTempo(asset.props); const fx = fxVoiceOf(project, pickedFx(fxId, asset.props), tempo, fxParams); + const bake = validBake(project, asset); return audio.playSequence( patternVoiceName(patternId), { @@ -398,6 +401,7 @@ function playPattern( ...(fx && { fx }), ...heardOn(playedRoute(project, bus, patternId)), strike: (note) => strikeNote({ project, audio }, { ...note, volume: level }, owner), + ...(bake && { bake: { file: bake.file, gain: bake.peak * level, playback: bakePlayback(bake, loop) } }), }, owner, ); @@ -406,8 +410,8 @@ function playPattern( /** * A song played by a game (#336): its tracks at their level, pan, mute and solo, at its tempo. With * `loop` it plays from its start and then runs round the loop of its ruler, else once to its end. - * `volume` and `fadeIn` are the level of the whole playing. Returns the number of the playing, 0 - * where there is no song or nothing in it to play. + * `volume` and `fadeIn` are the level of the whole playing. Its bake (#340) is played instead while + * it holds. Returns the number of the playing, 0 where there is no song or nothing in it to play. */ function playSong( { project, audio }: Pick, @@ -425,6 +429,7 @@ function playSong( const sequence = songSequence(project, song); const tempo = songTempo(song.props); const fx = fxVoiceOf(project, pickedFx(fxId, song.props), tempo, fxParams); + const bake = validBake(project, song); return audio.playSequence( songVoiceName(songId), { @@ -440,6 +445,7 @@ function playSong( ...heardOn(playedRoute(project, bus, songId)), strike: (note) => strikeNote({ project, audio }, note, owner), strikeClip: (clip) => playClip({ project, audio }, clip), + ...(bake && { bake: { file: bake.file, gain: bake.peak, playback: bakePlayback(bake, loop) } }), }, owner, ); diff --git a/src/viewport/SoundLibrary.ts b/src/viewport/SoundLibrary.ts index ca4f4f7d..19da8461 100644 --- a/src/viewport/SoundLibrary.ts +++ b/src/viewport/SoundLibrary.ts @@ -16,7 +16,14 @@ import { } from '../core/soundfx/nodes'; import { releaseTime } from '../core/instrument/envelope'; import type { EaseShare } from '../core/model/animation'; -import { type BusLevel, Playbacks, type SequenceNeeds, type SequencePlay } from '../core/music/playbacks'; +import { BAKE_RATE } from '../core/music/bake'; +import { + type BakePlay, + type BusLevel, + Playbacks, + type SequenceNeeds, + type SequencePlay, +} from '../core/music/playbacks'; import { arrivalOf, sendDelays, type TimedBus } from '../core/music/routing'; import type { SendTap } from '../core/graph/Graph'; import { MASTER_MIX, type MixBus, type MixSend, sameMix } from '../core/music/mixer'; @@ -160,6 +167,8 @@ interface Voice { /** How many of its ways play through the pitch worklet, and whether its note has been let go. */ vocoders: number; released: boolean; + /** Seconds a stop that lets it ring fades it out over: a bake (#340), whose notes are one sound. */ + ringFade: number; } /** @@ -180,7 +189,7 @@ type Way = Exclude; * its own plays (the sound a `play` was given), and what has been made for it so far. */ interface Wiring { - context: AudioContext; + context: BaseAudioContext; /** When its sources start on the audio clock. */ at: number; buffers: ReadonlyMap; @@ -256,7 +265,7 @@ interface Pending { */ export class SoundLibrary implements AudioSink { private base = DEFAULT_BASE; - private context: AudioContext | null = null; + private context: BaseAudioContext | null = null; /** * Where the master goes out (#359): through a limiter that keeps every sample at `GUARD_DB` or * below, once the worklet is there; straight to the speakers until then. Late by the limiter. @@ -264,9 +273,22 @@ export class SoundLibrary implements AudioSink { private guard: { input: GainNode; limiter: AudioWorkletNode | null; latency: number } | null = null; private guardMeter: BusMeter | null = null; - /** `guarded` false leaves the limiter out: for checks that follow every node of a voice. */ - constructor(private readonly guarded = true) {} - private offline: OfflineAudioContext | null = null; + /** + * `guarded` false leaves the limiter out: for checks that follow every node of a voice. With + * `render` it plays into an offline context to bake (#340), never resumed by itself. + */ + constructor( + private readonly guarded = true, + private readonly render: OfflineRender | null = null, + ) { + if (render) { + this.context = render.context; + this.held = true; + } + } + private decodeContext: OfflineAudioContext | null = null; + /** Where a bake (#340) is decoded: always at the rate it was made at, so its loop lands on the sample. */ + private opusContext: OfflineAudioContext | null = null; private readonly buffers = new Map(); /** * The stretches of files that sound assets are cut to (#343), made from the whole file on first @@ -320,12 +342,13 @@ export class SoundLibrary implements AudioSink { stop: (voices, letRing) => { for (const id of voices) this.stopVoice(id, letRing); }, - wake, + wake: (pump) => (this.render?.wake ?? wake)(pump), prepare: (needs, id) => this.ready(needs, id), ahead: (id) => this.groupAhead(id), level: (id, gain, at, seconds) => this.level(id, gain, at, seconds), after: (at, then) => void this.after(at, then), closed: () => this.sweep(), + playBaked: (id, bake, at) => this.playBaked(id, bake, at), }); /** The playings whose buses are still there: a master each, and the tracks of a song into it. */ private readonly groups = new Set(); @@ -833,7 +856,7 @@ export class SoundLibrary implements AudioSink { * a Mix is held back as far, so the two stay together. Says where it comes out and how late. */ private chainOf( - context: AudioContext, + context: BaseAudioContext, chain: FxChain, input: AudioNode, made: BusChain, @@ -897,7 +920,7 @@ export class SoundLibrary implements AudioSink { /** The Pitch of a way of a bus (#346); without the worklet the sound goes by as it is, on time. */ private shifter( - context: AudioContext, + context: BaseAudioContext, pitch: number, input: AudioNode, made: BusChain, @@ -953,15 +976,20 @@ export class SoundLibrary implements AudioSink { playSequence(name: string, play: SequencePlay, owner = ''): number { if (play.sequence.notes.length === 0 && !play.sequence.clips?.length) return 0; const id = this.nextId++; - const buses = play.buses ?? []; + // A bake whose file could not be had is no bake: the notes are played instead (#340). + const { bake: asked, ...rest } = play; + const bake = asked && !this.hasFailed(asked.file) ? asked : null; + const buses = bake ? [] : (play.buses ?? []); this.open(id, buses, play.fx ?? null, play.bus ?? MASTER_MIX, play.sends); const tracks = new Set(buses.map((bus) => bus.name)); const into = (bus: string | undefined) => groupBus(id, bus !== undefined && tracks.has(bus) ? bus : ''); const { strike, strikeClip } = play; const chains = [...buses.flatMap((bus) => bus.fx ?? []), ...(play.fx ? [play.fx] : [])]; + const needs = bake ? { layers: [], files: [bake.file] } : play.needs; const routed: SequencePlay = { - ...play, - needs: { ...play.needs, layers: [...play.needs.layers, ...chains.map((fx) => fx.plan)] }, + ...rest, + ...(bake && { bake }), + needs: { ...needs, layers: [...needs.layers, ...chains.map((fx) => fx.plan)] }, strike: (note) => strike({ ...note, bus: into(note.bus) }), ...(strikeClip && { strikeClip: (clip: ScheduledClip) => strikeClip({ ...clip, bus: into(clip.bus) }), @@ -980,7 +1008,10 @@ export class SoundLibrary implements AudioSink { else this.stopVoice(id); } - /** The patterns and songs playing, oldest first, with the notes and clips last planned of each (test hook). */ + /** + * The patterns and songs playing, oldest first, with the notes and clips last planned of each, and + * whether it plays its bake (test hook). + */ playbacks(): { id: number; name: string; @@ -989,18 +1020,128 @@ export class SoundLibrary implements AudioSink { fading: number | null; planned: ScheduledNote[]; clips: ScheduledClip[]; + baked: boolean; + startedAt: number | null; }[] { - return this.sequences.current().map(({ id, name, owner, scheduler, fading, planned, clips }) => ({ + return this.sequences.current().map(({ id, name, owner, runner, fading, planned, clips, baked }) => ({ id, name, owner, - playing: scheduler.playing, + playing: runner.playing, + startedAt: runner.startedAt, fading, planned: [...planned], clips: [...clips], + baked, })); } + /** + * What goes into the master of a playing, taken off before its level into `to` (#340): what a bake + * keeps. Says whether there was such a playing. + */ + tapPlaying(id: number, to: AudioNode): boolean { + const master = this.buses.get(groupBus(id, '')); + if (!master) return false; + master.input.disconnect(); + master.input.connect(to); + return true; + } + + /** + * The bake of a playing (#340) as one voice on its master, heard from `heard`: its file once to its + * end, faded out there, or up to the loop end and then round its second round – in one source where + * the round follows at once, else in a second that takes over to the sample. 0 without its bytes. + */ + private playBaked(id: number, bake: BakePlay, heard: number): number { + const buffer = this.buffer(bake.file); + if (!buffer) return 0; + const context = this.audio(); + const bus = groupBus(id, ''); + const at = Math.max(context.currentTime, heard - this.busLatency(bus)); + const gain = context.createGain(); + gain.gain.value = Math.max(0, Math.min(MAX_VOLUME, bake.gain)); + gain.connect(this.busInput(bus)); + const { end, until, round } = bake.playback; + const made = (from: number, loop: { start: number; end: number } | null) => { + const node = context.createBufferSource(); + node.buffer = buffer; + if (loop) { + node.loop = true; + node.loopStart = loop.start; + node.loopEnd = loop.end; + } + node.connect(gain); + const source: Source = { + start: () => {}, + stop: () => node.stop(), + ended: (then) => { + node.onended = then; + }, + fade: null, + gate: false, + leave: null, + }; + return { node, source, from }; + }; + const parts = + round && round.start === until + ? [made(0, round)] + : round + ? [made(0, null), made(round.start, round)] + : [made(0, null)]; + const voice: Voice = { + id: this.nextId++, + file: bake.file, + owner: '', + bus, + at, + sources: parts.map((part) => part.source), + ending: new Set(), + drivers: [], + envelopes: [], + finish: null, + gain, + fx: { layers: [], params: {}, tempo: DEFAULT_TEMPO }, + knobs: [], + rings: false, + ringing: false, + quiet: null, + vocoders: 0, + released: false, + ringFade: BAKE_LET_RING, + }; + const going = new Set(voice.sources); + voice.finish = (source) => { + if (going.delete(source) && going.size === 0) this.ended(voice); + }; + for (const { source } of parts) source.ended(() => voice.finish?.(source)); + this.voices.push(voice); + this.started += 1; + this.last = { + file: bake.file, + volume: gain.gain.value, + loop: round !== null, + layers: 0, + pitches: [], + stages: 0, + duration: buffer.duration, + at, + envelopes: 0, + bus, + }; + const [head, next] = parts; + if (end !== null) { + gain.gain.setValueAtTime(gain.gain.value, at + Math.max(0, end - FADE)); + gain.gain.linearRampToValueAtTime(0, at + end); + head!.node.start(at, 0, end); + } else if (next) { + head!.node.start(at, 0, until); + next.node.start(at + until, next.from); + } else head!.node.start(at, 0); + return voice.id; + } + /** * The buses of a playing, its tracks into its master and a pattern item's into its track (#346), * each with the sound FX it was given; its master into `into`, or along `sends` (#351). Their @@ -1302,7 +1443,7 @@ export class SoundLibrary implements AudioSink { */ setSuspended(suspended: boolean): void { this.held = suspended; - const context = this.context; + const context = this.live(); if (!context) return; if (suspended) void context.suspend(); else void context.resume(); @@ -1363,13 +1504,19 @@ export class SoundLibrary implements AudioSink { /** Seconds on the audio clock, opening the context where there is none yet: what the scheduler plans by (#332). */ time(): number { const context = this.audio(); - if (context.state === 'suspended' && !this.held) void context.resume(); + if (context.state === 'suspended' && !this.held) void this.live()?.resume(); return context.currentTime; } /** The context everything plays on, opened where there is none yet: a take is recorded on its clock (#342). */ liveContext(): AudioContext { - return this.audio(); + this.audio(); + return this.live()!; + } + + /** The context it plays to the speakers on; null before there is one, and for a bake (#340). */ + private live(): AudioContext | null { + return this.render ? null : (this.context as AudioContext | null); } /** @@ -1377,7 +1524,7 @@ export class SoundLibrary implements AudioSink { * device, where the browser says. */ latency(): number { - const context = this.context; + const context = this.live(); if (!context) return 0; const guard = this.guard?.latency ?? 0; // Older Safari and Firefox have no `outputLatency`, some no `baseLatency` either. @@ -1399,6 +1546,11 @@ export class SoundLibrary implements AudioSink { ]); } + /** Loads the FX worklet into its context, for a node of it made outside (#340); says whether it is there. */ + worklet(): Promise { + return this.fxWorklet(); + } + /** Where the FX worklet is served from, for a check that runs it in an offline context (test hook). */ fxWorkletUrl(): string { return fxWorkletUrl; @@ -1475,10 +1627,8 @@ export class SoundLibrary implements AudioSink { if (running) return running; const work = (async () => { try { - const response = await fetch(`${this.base}/${file}`); - if (!response.ok) throw new Error(`HTTP ${response.status}`); - const bytes = await response.arrayBuffer(); - const buffer = await this.decoder().decodeAudioData(bytes); + const buffer = this.render ? await this.render.files.load(file) : await this.fetched(file); + if (!buffer) throw new Error('not there where it is baked from'); this.buffers.set(file, buffer); return buffer; } catch (error: unknown) { @@ -1493,6 +1643,12 @@ export class SoundLibrary implements AudioSink { return work; } + private async fetched(file: string): Promise { + const response = await fetch(`${this.base}/${file}`); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + return this.decoder(file).decodeAudioData(await response.arrayBuffer()); + } + /** * Ends these voices now: `onended` only fires next frame and the counts have to be right here. A * voice is faded out rather than cut, so it does not click. With `letRing` one with echoes only @@ -1502,6 +1658,10 @@ export class SoundLibrary implements AudioSink { private silence(going: Voice[], letRing = false): void { this.stopped += going.filter((voice) => !(letRing && voice.ringing)).length; for (const voice of going) { + if (letRing && voice.ringFade > 0) { + this.cut(voice, voice.ringFade); + continue; + } if (letRing && voice.envelopes.length > 0) { this.letGo(voice, this.audio().currentTime, true); continue; @@ -1530,8 +1690,8 @@ export class SoundLibrary implements AudioSink { } } - /** Fades the whole voice out and lets it go. */ - private cut(voice: Voice): void { + /** Fades the whole voice out over `seconds` and lets it go. */ + private cut(voice: Voice, seconds = FADE): void { this.forget(voice); const context = this.audio(); const done = () => { @@ -1539,10 +1699,11 @@ export class SoundLibrary implements AudioSink { for (const driver of voice.drivers) driver.stop(); voice.gain.disconnect(); }; - // A suspended clock would hold the fade – and the sound – until the next run resumes it. - if (context.state !== 'running') return done(); - fadeOut(voice.gain.gain, context.currentTime); - this.after(context.currentTime + FADE, done); + // A suspended clock would hold the fade – and the sound – until the next run resumes it. An + // offline one is suspended whenever the page gets to plan, and runs on after. + if (context.state !== 'running' && !this.render) return done(); + fadeOut(voice.gain.gain, context.currentTime, seconds); + this.after(context.currentTime + seconds, done); } /** Fades the sources of a voice with echoes out and lets the echoes ring on (#324). */ @@ -1649,7 +1810,7 @@ export class SoundLibrary implements AudioSink { const context = this.audio(); // A context starts suspended where the page has not been interacted with; without this the // voice would run silently and end as if it had played. - if (context.state === 'suspended' && !this.held) void context.resume(); + if (context.state === 'suspended' && !this.held) void this.live()?.resume(); // As much earlier as its bus and its own stages are late, so it is heard when it was planned // (#346, #326). A layer that is less late is held back to the latest. const chains = fx.layers.map((plan) => fxChainOf(plan, fx.params, fx.tempo)); @@ -1721,6 +1882,7 @@ export class SoundLibrary implements AudioSink { quiet: null, vocoders: wiring.vocoders, released: false, + ringFade: 0, }; const going = new Set(sources); voice.finish = (source) => { @@ -1967,9 +2129,10 @@ export class SoundLibrary implements AudioSink { return glideKnobs(voice.knobs, levels, this.audio().currentTime, time, ease); } - private audio(): AudioContext { + private audio(): BaseAudioContext { if (this.context) return this.context; - const context = (this.context = new AudioContext()); + const context = new AudioContext(); + this.context = context; if (this.guarded) this.guardOut(context); if (this.mix.length > 0) void this.applyMix(); return context; @@ -2015,7 +2178,7 @@ export class SoundLibrary implements AudioSink { } const range = this.rangeOf(name); if (!range) return null; - const made = this.decoder().createBuffer( + const made = this.decoder(name).createBuffer( range.whole.numberOfChannels, range.to - range.from, range.whole.sampleRate, @@ -2073,10 +2236,16 @@ export class SoundLibrary implements AudioSink { })()); } - /** The context that is already playing, or else one that decodes without opening a device. */ - private decoder(): BaseAudioContext { - if (this.context || typeof OfflineAudioContext === 'undefined') return this.audio(); - return (this.offline ??= new OfflineAudioContext(1, 1, DECODE_RATE)); + /** + * The context that is already playing, or else one that decodes without opening a device. An Ogg + * file, which a bake (#340) is, is decoded at the rate Opus is made at, whatever the device runs + * at: resampled, the loop of a bake would lose a sample a round. + */ + private decoder(file: string): BaseAudioContext { + if (typeof OfflineAudioContext === 'undefined') return this.audio(); + if (file.endsWith('.ogg')) return (this.opusContext ??= new OfflineAudioContext(1, 1, BAKE_RATE)); + if (this.context) return this.audio(); + return (this.decodeContext ??= new OfflineAudioContext(1, 1, DECODE_RATE)); } /** Everything that would have silenced this file for this owner, counted rather than flagged. */ @@ -2085,6 +2254,19 @@ export class SoundLibrary implements AudioSink { } } +/** + * What a library that bakes (#340) runs on: an offline context, the library whose decoded files it + * plays, and what wakes its schedulers – the offline clock, not the page's timers. + */ +export interface OfflineRender { + context: OfflineAudioContext; + files: SoundLibrary; + wake(pump: () => void): () => void; +} + +/** Seconds a bake fades out over where its notes would have rung on (#340). */ +const BAKE_LET_RING = 0.3; + /** The bus of a game's playing (#336): its master for `''`, else the track of that name inside it. */ function groupBus(id: number, track: string): string { return track === '' ? `play:${id}` : `play:${id}:${track}`; diff --git a/src/viewport/bakeRender.ts b/src/viewport/bakeRender.ts new file mode 100644 index 00000000..e3cc7351 --- /dev/null +++ b/src/viewport/bakeRender.ts @@ -0,0 +1,232 @@ +/** + * Bakes a pattern or a song (#340): plays it into an offline context with a library of its own – + * the same code a game hears it through, its scheduler woken by the offline clock – takes what + * reaches its master, and encodes that as Opus in Ogg with WebCodecs. Chrome has all of it. + */ +import { + assembleBake, + BAKE_RATE, + type Bake, + type BakedAsset, + bakePrint, + bakeRuns, + type BakeSource, + bakeSourceOf, + bakeSpans, + normalizeBake, +} from '../core/music/bake'; +import { sequenceNeeds } from '../core/music/needs'; +import { oggOpus, OPUS_RATE, type OpusPacket, preSkipOf } from '../core/music/ogg'; +import type { BakePlay } from '../core/music/playbacks'; +import type { Project } from '../core/model/Project'; +import { playClip, strikeNote } from '../runtime/systems/behaviorContext'; +import { SoundLibrary, soundLibrary } from './SoundLibrary'; + +/** Ten render quanta between two plans: about as often as the page plans (`INTERVAL`). */ +const STEP = 1280; +/** Seconds of room before the first note is heard: the lead of a start and how late its buses are. */ +const ROOM = 2; +/** Frames handed to the encoder at once. */ +const BLOCK = 48000; +/** What libopus runs ahead at 48 kHz, where the encoder does not say. */ +const DEFAULT_PRE_SKIP = 312; +const STEREO_BITRATE = 160_000; +const MONO_BITRATE = 96_000; + +export class BakeCancelled extends Error { + constructor() { + super('Baking was cancelled'); + } +} + +/** Says how far it has got, 0 to 1, and whether it is to give up. */ +export interface BakeWatch { + progress(share: number): void; + cancelled(): boolean; +} + +/** A bake as it is made, before its file has a name. */ +export type MadeBake = Omit & { data: Uint8Array }; + +/** The file of a bake and what goes with it; throws `BakeCancelled`, or why it cannot be made. */ +export async function makeBake(project: Project, asset: BakedAsset, watch: BakeWatch): Promise { + const source = bakeSourceOf(project, asset); + if (!source) throw new Error(`${asset.name} plays nothing, so there is nothing to bake`); + const print = bakePrint(project, asset); + const spans = bakeSpans(source); + const runs = bakeRuns(spans); + const total = runs.looped + (runs.once ?? 0); + const share = (from: number, frames: number) => (part: number) => + watch.progress((0.85 * (from + part * frames)) / total); + const looped = await renderRun(project, source, true, runs.looped, null, { + ...watch, + progress: share(0, runs.looped), + }); + const once = + runs.once === null + ? null + : await renderRun(project, source, false, runs.once, null, { + ...watch, + progress: share(runs.looped, runs.once), + }); + // The master hears mono only where nothing on the way made it stereo; then the bake is mono too. + const mono = looped.mono && (once?.mono ?? true); + const sides = (run: Rendered) => (mono ? run.channels.slice(0, 1) : run.channels); + const channels = assembleBake(spans, sides(looped), once && sides(once)); + const peak = normalizeBake(channels); + const data = await encodeOpus(channels, { + ...watch, + progress: (part) => watch.progress(0.85 + 0.15 * part), + }); + return { print, peak, ...spans, data }; +} + +export interface Rendered { + channels: Float32Array[]; + /** Whether what reached the master was mono all along. */ + mono: boolean; +} + +/** + * `frames` of the source as they reach its master, from the moment its first note is heard: played + * round its loop or once, from its notes – or from `bake` where one is given (what a check holds + * against the notes). + */ +export async function renderRun( + project: Project, + source: BakeSource, + loop: boolean, + frames: number, + bake: BakePlay | null, + watch: BakeWatch, +): Promise { + const context = new OfflineAudioContext(2, frames + ROOM * BAKE_RATE, BAKE_RATE); + const pumps = new Set<() => void>(); + const library = new SoundLibrary(false, { + context, + files: soundLibrary, + wake: (pump) => { + pumps.add(pump); + return () => pumps.delete(pump); + }, + }); + const sink = { project, audio: library }; + const id = library.playSequence('bake', { + sequence: source.sequence, + tempo: source.tempo, + length: source.length, + loop: loop ? source.loop : null, + needs: sequenceNeeds(project, source.sequence), + buses: source.buses, + strike: (note) => strikeNote(sink, note, ''), + strikeClip: (clip) => playClip(sink, clip), + ...(bake && { bake }), + }); + const capture = context.createGain(); + capture.connect(context.destination); + library.tapPlaying(id, capture); + let most = 2; + if (await library.worklet()) { + most = 0; + const probe = new AudioWorkletNode(context, 'channel-probe', { numberOfOutputs: 0 }); + probe.port.onmessage = (event: MessageEvent) => { + most = Math.max(most, event.data); + }; + capture.connect(probe); + } + const anchor = await startOf(library, watch); + const offset = Math.round(anchor * BAKE_RATE); + if (offset + frames > context.length) throw new Error('Its buses hold it back too far to bake'); + let frame = 0; + const hop = () => { + frame += STEP; + if (frame >= context.length) return; + void context.suspend(frame / BAKE_RATE).then(() => { + if (!watch.cancelled()) for (const pump of pumps) pump(); + watch.progress(Math.min(1, frame / (offset + frames))); + hop(); + void context.resume(); + }); + }; + hop(); + const rendered = await context.startRendering(); + // What the probe posted last is on its way still. + await new Promise((resolve) => setTimeout(resolve, 0)); + if (watch.cancelled()) throw new BakeCancelled(); + return { + channels: [0, 1].map((c) => rendered.getChannelData(c).subarray(offset, offset + frames)), + mono: most <= 1, + }; +} + +/** The second of the offline clock its first note is heard at, once what it needs is loaded. */ +async function startOf(library: SoundLibrary, watch: BakeWatch): Promise { + for (;;) { + const started = library.playbacks()[0]?.startedAt; + if (started !== undefined && started !== null) return started; + if (watch.cancelled()) throw new BakeCancelled(); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +/** Opus in Ogg of these samples at `OPUS_RATE`, one or two channels. */ +export async function encodeOpus(channels: readonly Float32Array[], watch: BakeWatch): Promise { + const config: AudioEncoderConfig = { + codec: 'opus', + sampleRate: OPUS_RATE, + numberOfChannels: channels.length, + bitrate: channels.length > 1 ? STEREO_BITRATE : MONO_BITRATE, + }; + const supported = + typeof AudioEncoder !== 'undefined' && + (await AudioEncoder.isConfigSupported(config).then( + (support) => support.supported === true, + () => false, + )); + if (!supported) throw new Error('This browser cannot encode Opus – baking needs Chrome'); + const packets: OpusPacket[] = []; + // Set from the encoder's callbacks, which the compiler cannot follow into. + const heard: { preSkip: number | null; failure: Error | null } = { preSkip: null, failure: null }; + const encoder = new AudioEncoder({ + output: (chunk, meta) => { + const head = meta?.decoderConfig?.description; + if (head && heard.preSkip === null) heard.preSkip = preSkipOf(bytesOf(head)); + const data = new Uint8Array(chunk.byteLength); + chunk.copyTo(data); + packets.push({ data, samples: Math.round(((chunk.duration ?? 0) * OPUS_RATE) / 1e6) }); + }, + error: (error) => { + heard.failure = error; + }, + }); + encoder.configure(config); + const frames = channels[0]?.length ?? 0; + for (let at = 0; at < frames && heard.failure === null && !watch.cancelled(); at += BLOCK) { + const count = Math.min(BLOCK, frames - at); + const planar = new Float32Array(count * channels.length); + channels.forEach((side, c) => planar.set(side.subarray(at, at + count), c * count)); + const data = new AudioData({ + format: 'f32-planar', + sampleRate: OPUS_RATE, + numberOfFrames: count, + numberOfChannels: channels.length, + timestamp: Math.round((at * 1e6) / OPUS_RATE), + data: planar, + }); + encoder.encode(data); + data.close(); + while (encoder.encodeQueueSize > 4) await new Promise((resolve) => setTimeout(resolve, 0)); + watch.progress(at / frames); + } + if (heard.failure === null && !watch.cancelled()) await encoder.flush(); + encoder.close(); + if (heard.failure !== null) throw heard.failure; + if (watch.cancelled()) throw new BakeCancelled(); + return oggOpus({ channels: channels.length, preSkip: heard.preSkip ?? DEFAULT_PRE_SKIP, frames, packets }); +} + +function bytesOf(source: AllowSharedBufferSource): Uint8Array { + return ArrayBuffer.isView(source) + ? new Uint8Array(source.buffer, source.byteOffset, source.byteLength) + : new Uint8Array(source); +} diff --git a/src/viewport/fxNodes.ts b/src/viewport/fxNodes.ts index 0d370222..cd0fb622 100644 --- a/src/viewport/fxNodes.ts +++ b/src/viewport/fxNodes.ts @@ -626,11 +626,11 @@ export function heldBack(context: BaseAudioContext, node: AudioNode, seconds: nu return node.connect(line); } -/** Sends a level to silence in `FADE` seconds from wherever it is now. */ -export function fadeOut(param: AudioParam, now: number): void { +/** Sends a level to silence in `seconds` from wherever it is now. */ +export function fadeOut(param: AudioParam, now: number, seconds = FADE): void { param.cancelScheduledValues(now); param.setValueAtTime(param.value, now); - param.linearRampToValueAtTime(0, now + FADE); + param.linearRampToValueAtTime(0, now + seconds); } /** diff --git a/src/viewport/fxWorklet.ts b/src/viewport/fxWorklet.ts index d9b5986e..e3841bc8 100644 --- a/src/viewport/fxWorklet.ts +++ b/src/viewport/fxWorklet.ts @@ -327,3 +327,22 @@ class ChipProcessor extends Source { } registerProcessor('chip-player', ChipProcessor); + +/** + * Says how many channels reach it at most (#340): whether what goes into the master of a pattern + * being baked is mono, so its bake is too and goes on the same way. + */ +class ChannelProbe extends AudioWorkletProcessor { + private most = 0; + + process(inputs: Float32Array[][]): boolean { + const count = inputs[0]?.length ?? 0; + if (count > this.most) { + this.most = count; + this.port.postMessage(count); + } + return true; + } +} + +registerProcessor('channel-probe', ChannelProbe); diff --git a/tests/core/bake.test.ts b/tests/core/bake.test.ts new file mode 100644 index 00000000..8e3502a2 --- /dev/null +++ b/tests/core/bake.test.ts @@ -0,0 +1,336 @@ +/** + * A pattern or song baked (#340): where the parts of its file lie, how it is put together from the + * playing looped and the one played once, raised to full scale, how it is played back – and what it + * is made from, so a change to anything it sounds by outdates it, and an undo brings it back. + */ +import { describe, expect, it } from 'vitest'; +import { + CreateAssetCommand, + CreateSoundCommand, + RenameAssetCommand, + SetAssetPropertyCommand, +} from '../../src/core/commands/assetCommands'; +import { AddClipCommand, SetClipPropsCommand } from '../../src/core/commands/clipCommands'; +import { SetNodeParamCommand } from '../../src/core/commands/graphCommands'; +import { SetPatternNotesCommand } from '../../src/core/commands/patternCommands'; +import { SetSongCommand } from '../../src/core/commands/songCommands'; +import type { InstrumentAsset, PatternAsset, SongAsset } from '../../src/core/model/Asset'; +import type { PropertyBag } from '../../src/core/model/Properties'; +import { projectFiles } from '../../src/core/model/projectFiles'; +import { + assembleBake, + type Bake, + bakePlayback, + bakePrint, + bakeRuns, + bakeSpans, + bakeUploadName, + heldBake, + isBakeUpload, + normalizeBake, + validBake, +} from '../../src/core/music/bake'; +import { type BakePlay, Playbacks, type SequencePlay } from '../../src/core/music/playbacks'; +import { LEAD } from '../../src/core/music/Scheduler'; +import { newItem } from '../../src/core/music/song'; +import { deserializeProject, serializeProject } from '../../src/core/store/serialize'; +import { SILENT_AUDIO } from '../../src/runtime/systems/Audio'; +import { graphEditor, makeBehavior, makeSpriteScene, place, startWorld } from '../helpers'; + +const Q = 480; +const FILE = `${'a1'.repeat(32)}.wav`; +const OTHER = `${'b2'.repeat(32)}.wav`; +const BAKED = `${'c3'.repeat(6)}/${'d4'.repeat(32)}.ogg`; + +/** A pattern of two notes over two bars at 90 BPM, through an instrument of one sample. */ +function baked() { + const f = makeSpriteScene('Box'); + const sound = new CreateSoundCommand('Blip', FILE); + f.stack.execute(sound); + const other = new CreateSoundCommand('Bell', OTHER); + f.stack.execute(other); + const ins = new CreateAssetCommand('instrument', 'Lead'); + f.stack.execute(ins); + const lead = ins.asset as InstrumentAsset; + const sample = lead.graph.nodes.find((n) => n.type === 'InstrumentSample')!; + sample.params.sound = sound.asset.id; + const made = new CreateAssetCommand('pattern', 'Tune'); + f.stack.execute(made); + const id = made.asset.id; + f.stack.execute(new SetAssetPropertyCommand(id, 'instrument', lead.id)); + f.stack.execute(new SetAssetPropertyCommand(id, 'tempo', 90)); + f.stack.execute(new SetAssetPropertyCommand(id, 'bars', 2)); + f.stack.execute( + new SetPatternNotesCommand(id, [ + { id: 'n1', tick: 0, length: Q, note: 60, velocity: 0.5 }, + { id: 'n2', tick: Q, length: Q, note: 64, velocity: 1 }, + ]), + ); + const pattern = () => f.project.assets.find((a) => a.id === id) as PatternAsset; + return { ...f, id, lead, sample, sound: sound.asset, other: other.asset, pattern }; +} + +function bakeOf(print: string, fields: Partial = {}): Bake { + return { + file: BAKED, + print, + peak: 0.5, + length: 100, + loopEnd: 100, + head: 100, + round: 100, + bytes: 1234, + ...fields, + }; +} + +describe('the parts of a bake', () => { + it('lies in frames at 48 kHz: the straight run as far as the end or the loop end, then one round', () => { + // Two bars of 4/4 at 120 BPM are 4 s; a loop of the second bar. + const source = { tempo: 120, length: 8 * Q, loop: { start: 4 * Q, end: 8 * Q } }; + expect(bakeSpans(source)).toEqual({ length: 192000, loopEnd: 192000, head: 192000, round: 96000 }); + const short = { ...source, loop: { start: 0, end: 4 * Q } }; + expect(bakeSpans(short)).toEqual({ length: 192000, loopEnd: 96000, head: 192000, round: 96000 }); + }); + + it('is played looped once past the loop end; once only where that goes further', () => { + expect(bakeRuns({ length: 100, loopEnd: 100, round: 100 })).toEqual({ looped: 200, once: null }); + expect(bakeRuns({ length: 300, loopEnd: 100, round: 50 })).toEqual({ looped: 150, once: 300 }); + }); + + it('takes the head from the playing once where there is one, the round always from the looped one', () => { + const ramp = (length: number, from: number) => Float32Array.from({ length }, (_, i) => from + i); + const looped = [ramp(150, 0), ramp(150, 1000)]; + const out = assembleBake({ loopEnd: 100, head: 100, round: 50 }, looped, null); + expect([out[0]!.length, out[0]![99], out[0]![100], out[1]![149]]).toEqual([150, 99, 100, 1149]); + const once = [ramp(300, 5000), ramp(300, 6000)]; + const both = assembleBake({ loopEnd: 100, head: 300, round: 50 }, looped, once); + // The round follows the head and starts where the looped playing passed its loop end. + expect([both[0]!.length, both[0]![299], both[0]![300], both[0]![349]]).toEqual([350, 5299, 100, 149]); + }); + + it('is raised to full scale, and played back at the peak it had', () => { + const sides = [Float32Array.of(0.25, -0.5), Float32Array.of(0.125, 0)]; + expect(normalizeBake(sides)).toBe(0.5); + expect([...sides[0]!, ...sides[1]!]).toEqual([0.5, -1, 0.25, 0]); + expect(normalizeBake([new Float32Array(3)])).toBe(0); + }); + + it('plays once to its end, or to the loop end and then round its round', () => { + const spans = { length: 96000, loopEnd: 96000, head: 96000, round: 48000 }; + expect(bakePlayback(spans, false)).toEqual({ end: 2, until: 2, round: null }); + expect(bakePlayback(spans, true)).toEqual({ end: null, until: 2, round: { start: 2, end: 3 } }); + expect(bakePlayback({ ...spans, length: 144000, head: 144000 }, true)).toEqual({ + end: null, + until: 2, + round: { start: 3, end: 4 }, + }); + }); +}); + +describe('what a bake is made from', () => { + it('stays the same through a rename, changes with a note and comes back with the undo', () => { + const f = baked(); + const print = bakePrint(f.project, f.pattern()); + expect(print).not.toBe(''); + f.stack.execute(new RenameAssetCommand(f.id, 'Other tune')); + expect(bakePrint(f.project, f.pattern())).toBe(print); + f.stack.execute( + new SetPatternNotesCommand(f.id, [{ id: 'n1', tick: 0, length: Q, note: 60, velocity: 0.75 }]), + ); + expect(bakePrint(f.project, f.pattern())).not.toBe(print); + f.stack.undo(); + expect(bakePrint(f.project, f.pattern())).toBe(print); + }); + + it('changes with what the pattern only names: the instrument and the level of its sound', () => { + const f = baked(); + const print = bakePrint(f.project, f.pattern()); + f.stack.execute(new SetNodeParamCommand(f.lead.id, f.sample.id, 'sound', f.other.id)); + expect(bakePrint(f.project, f.pattern())).not.toBe(print); + f.stack.undo(); + expect(bakePrint(f.project, f.pattern())).toBe(print); + f.stack.execute(new SetAssetPropertyCommand(f.sound.id, 'volume', 0.5)); + expect(bakePrint(f.project, f.pattern())).not.toBe(print); + }); + + it('changes with the cut of a clip on a track of a song, and comes back with the undo', () => { + const f = baked(); + const clip = new AddClipCommand(f.sound.id, { start: 0.25, end: 1 }); + f.stack.execute(clip); + const made = new CreateAssetCommand('song', 'Theme'); + f.stack.execute(made); + const song = () => f.project.assets.find((a) => a.id === made.asset.id) as SongAsset; + const track = song().tracks[0]!.id; + const item = newItem({ track, kind: 'sound', asset: clip.clipId, tick: 0, length: 4 * Q }); + f.stack.execute(new SetSongCommand(song().id, { items: [item] })); + const print = bakePrint(f.project, song()); + expect(print).not.toBe(''); + const changes: PropertyBag[] = [ + { end: 0.75 }, + { reverse: true }, + { fadeOut: 50 }, + { fadeInCurve: 'sCurve' }, + ]; + for (const change of changes) { + f.stack.execute(new SetClipPropsCommand(f.sound.id, clip.clipId, change)); + expect(bakePrint(f.project, song()), JSON.stringify(change)).not.toBe(print); + f.stack.undo(); + expect(bakePrint(f.project, song())).toBe(print); + } + }); + + it('is nothing for a pattern that plays nothing', () => { + const f = baked(); + f.stack.execute(new SetAssetPropertyCommand(f.id, 'instrument', '')); + expect(bakePrint(f.project, f.pattern())).toBe(''); + }); + + it('holds while the print is the same, and only then is its file loaded with the project', () => { + const f = baked(); + f.pattern().bake = bakeOf(bakePrint(f.project, f.pattern())); + expect(validBake(f.project, f.pattern())?.file).toBe(BAKED); + expect(projectFiles(f.project).sounds).toContain(BAKED); + f.stack.execute(new SetAssetPropertyCommand(f.id, 'tempo', 100)); + expect(validBake(f.project, f.pattern())).toBeNull(); + expect(projectFiles(f.project).sounds).not.toContain(BAKED); + }); +}); + +describe('a bake kept with the project', () => { + it('is saved and loaded with its pattern; one that does not fit together is dropped', () => { + const f = baked(); + f.pattern().bake = bakeOf('x'); + const back = deserializeProject(serializeProject(f.project)); + expect((back.assets.find((a) => a.id === f.id) as PatternAsset).bake).toEqual(bakeOf('x')); + expect(heldBake(bakeOf('x', { head: 50 }))).toBeNull(); + expect(heldBake(bakeOf('x', { round: 101 }))).toBeNull(); + expect(heldBake(bakeOf('x', { file: 'x.ogg' }))).toBeNull(); + expect(heldBake(bakeOf(''))).toBeNull(); + }); + + it('goes up under its name behind a mark the sound picker leaves out by, a slash no folder', () => { + expect(bakeUploadName('Boss / theme')).toBe('[bake] Boss - theme.ogg'); + expect([isBakeUpload('[bake] Boss - theme.ogg'), isBakeUpload('Boss.ogg')]).toEqual([true, false]); + }); +}); + +describe('Play pattern with a bake', () => { + function played(loop: boolean) { + const f = baked(); + const behavior = makeBehavior(f, 'Logic'); + const box = place(f, f.spriteId, { x: 0, y: 0 }); + place(f, behavior.id, { x: 0, y: 0 }, box); + const edit = graphEditor(f, behavior); + edit.connect( + edit.addNode('OnStart'), + 'exec', + edit.addNode('PlayPattern', { pattern: f.id, volume: 0.5, loop }), + 'exec', + ); + const asked: SequencePlay[] = []; + const audio = { + ...SILENT_AUDIO, + playSequence: (_name: string, play: SequencePlay) => (asked.push(play), 1), + }; + return { ...f, run: () => startWorld(f, { behaviors: true, audio }).frame(), asked }; + } + + it('hands the bake over while it holds, at its peak times the volume of the node', () => { + const f = played(true); + const length = 2 * 4 * Q; + const spans = bakeSpans({ tempo: 90, length, loop: { start: 0, end: length } }); + f.pattern().bake = bakeOf(bakePrint(f.project, f.pattern()), spans); + f.run(); + expect(f.asked[0]?.bake).toEqual({ file: BAKED, gain: 0.25, playback: bakePlayback(spans, true) }); + expect(f.asked[0]?.sequence.notes).toHaveLength(2); + }); + + it('hands over none once it is outdated, and the notes are played', () => { + const f = played(false); + f.pattern().bake = bakeOf('outdated'); + f.run(); + expect(f.asked[0]?.bake).toBeUndefined(); + expect(f.asked[0]?.loop).toBeNull(); + }); +}); + +describe('a playing that plays its bake', () => { + function host() { + const made = { + time: 10, + sounding: new Set(), + baked: [] as { id: number; bake: BakePlay; at: number }[], + stopped: [] as { voices: number[]; letRing: boolean }[], + pumps: 0, + loading: [] as (() => void)[], + }; + const playbacks = new Playbacks({ + now: () => made.time, + latency: () => 0, + playing: (voice) => made.sounding.has(voice), + stop: (voices, letRing = false) => { + made.stopped.push({ voices: [...voices], letRing }); + for (const voice of voices) made.sounding.delete(voice); + }, + wake: () => { + made.pumps += 1; + return () => {}; + }, + prepare: () => new Promise((resolve) => made.loading.push(resolve)), + ahead: () => 0.25, + level: () => {}, + after: () => {}, + playBaked: (id, bake, at) => { + made.baked.push({ id, bake, at }); + made.sounding.add(500); + return 500; + }, + }); + const loaded = async () => { + for (const resolve of made.loading.splice(0)) resolve(); + await Promise.resolve(); + }; + return { made, playbacks, loaded }; + } + + const play = (end: number | null): SequencePlay => ({ + sequence: { notes: [{ tick: 0, length: Q, note: 60, velocity: 1, instrument: 'lead' }] }, + tempo: 120, + length: 4 * Q, + loop: end === null ? { start: 0, end: 4 * Q } : null, + needs: { layers: [], files: [BAKED] }, + strike: () => { + throw new Error('a baked playing strikes no note'); + }, + bake: { + file: BAKED, + gain: 1, + playback: { end, until: 2, round: end === null ? { start: 2, end: 4 } : null }, + }, + }); + + it('starts one voice of it once its file is there, as late as its first note would be, waking nothing', async () => { + const { made, playbacks, loaded } = host(); + playbacks.start(7, 'pattern:a', play(null), ''); + expect(made.baked).toEqual([]); + await loaded(); + expect(made.baked.map(({ id, at }) => ({ id, at }))).toEqual([{ id: 7, at: 10 + LEAD + 0.25 }]); + expect(made.pumps).toBe(0); + expect(playbacks.current().map((p) => [p.id, p.baked, p.runner.startedAt])).toEqual([ + [7, true, 10 + LEAD + 0.25], + ]); + playbacks.stopId(7, true); + expect(made.stopped).toEqual([{ voices: [500], letRing: true }]); + }); + + it('played once, it is gone with its voice; looped it stays', async () => { + const { made, playbacks, loaded } = host(); + playbacks.start(7, 'pattern:a', play(2), ''); + playbacks.start(8, 'pattern:b', play(null), ''); + await loaded(); + expect(playbacks.has(7)).toBe(true); + made.sounding.clear(); + expect(playbacks.current().map((p) => p.id)).toEqual([8]); + }); +}); diff --git a/tests/core/ogg.test.ts b/tests/core/ogg.test.ts new file mode 100644 index 00000000..66910c81 --- /dev/null +++ b/tests/core/ogg.test.ts @@ -0,0 +1,67 @@ +/** + * Opus in Ogg as a bake is kept (#340): the pages around the packets, their checksums, where each + * page says it has got to, and the pre-skip and length a decoder trims by. + */ +import { describe, expect, it } from 'vitest'; +import { oggCrc, oggOpus, opusHead, preSkipOf, readOggOpus } from '../../src/core/music/ogg'; + +const packet = (size: number, fill: number) => ({ data: new Uint8Array(size).fill(fill), samples: 960 }); + +/** The flags and the granule position of every page, in order. */ +function pagesOf(file: Uint8Array): { flags: number; granule: number }[] { + const view = new DataView(file.buffer); + const pages: { flags: number; granule: number }[] = []; + for (let at = 0; at < file.length;) { + pages.push({ flags: file[at + 5]!, granule: view.getUint32(at + 6, true) }); + const count = file[at + 26]!; + at += 27 + count + file.subarray(at + 27, at + 27 + count).reduce((s, v) => s + v, 0); + } + return pages; +} + +describe('Opus in Ogg', () => { + it('checks a page with the CRC Ogg uses', () => { + expect(oggCrc(Uint8Array.from('123456789', (c) => c.charCodeAt(0)))).toBe(0x89a1897f); + }); + + it('reads back the head, the packets and the length it was given', () => { + const packets = [packet(3, 1), packet(255, 2), packet(600, 3), packet(0, 4)]; + const file = oggOpus({ channels: 2, preSkip: 312, frames: 3000, packets }); + const back = readOggOpus(file); + expect(back.channels).toBe(2); + expect(back.preSkip).toBe(312); + expect(back.granule).toBe(312 + 3000); + expect(back.packets.map((p) => [p.length, p[0] ?? -1])).toEqual([ + [3, 1], + [255, 2], + [600, 3], + [0, -1], + ]); + }); + + it('opens with the head alone, flags the first and the last page', () => { + const file = oggOpus({ channels: 1, preSkip: 312, frames: 960, packets: [packet(10, 1)] }); + expect(String.fromCharCode(...file.subarray(28, 36))).toBe('OpusHead'); + expect(pagesOf(file).map((page) => page.flags)).toEqual([0x02, 0x00, 0x04]); + }); + + it('starts a new page before 255 lacing values, each counting the samples finished on it', () => { + const packets = Array.from({ length: 300 }, (_, i) => packet(100, i % 256)); + const file = oggOpus({ channels: 2, preSkip: 312, frames: 300 * 960 - 500, packets }); + const back = readOggOpus(file); + expect(back.packets).toHaveLength(300); + expect(back.pages).toBe(4); + expect(pagesOf(file).map((page) => page.granule)).toEqual([0, 0, 255 * 960, 312 + 300 * 960 - 500]); + }); + + it('refuses a page whose bytes were changed', () => { + const file = oggOpus({ channels: 2, preSkip: 312, frames: 960, packets: [packet(20, 7)] }); + file[file.length - 3] = file[file.length - 3]! ^ 0xff; + expect(() => readOggOpus(file)).toThrow(/checksum/); + }); + + it('reads the pre-skip of a head an encoder hands out', () => { + expect(preSkipOf(opusHead(2, 3840))).toBe(3840); + expect(preSkipOf(new Uint8Array(19))).toBeNull(); + }); +}); diff --git a/tests/core/song-play.test.ts b/tests/core/song-play.test.ts index aa61aee0..e96a56f0 100644 --- a/tests/core/song-play.test.ts +++ b/tests/core/song-play.test.ts @@ -11,6 +11,7 @@ import { import { SetLoopCommand, SetPatternNotesCommand } from '../../src/core/commands/patternCommands'; import { SetSongCommand } from '../../src/core/commands/songCommands'; import type { InstrumentAsset, SongAsset } from '../../src/core/model/Asset'; +import { bakePlayback, bakePrint, bakeSpans } from '../../src/core/music/bake'; import { goesBy, Playbacks, type SequencePlay } from '../../src/core/music/playbacks'; import { INTERVAL, LEAD, type ScheduledClip, type ScheduledNote } from '../../src/core/music/Scheduler'; import { newItem, newTrack, trackBus } from '../../src/core/music/song'; @@ -264,6 +265,27 @@ function recordingAudio() { } describe('Play song', () => { + it('hands over its bake while it holds, at the peak it had, its volume left live (#340)', () => { + const f = playingSong(); + const song = f.project.assets.find((a) => a.id === f.songId) as SongAsset; + const spans = bakeSpans({ tempo: 90, length: 2 * BAR, loop: { start: BAR, end: 2 * BAR } }); + const file = `${'c3'.repeat(6)}/${'d4'.repeat(32)}.ogg`; + song.bake = { file, print: bakePrint(f.project, song), peak: 0.5, bytes: 1, ...spans }; + const audio = recordingAudio(); + startWorld(f, { behaviors: true, audio }).frame(); + expect(audio.sequences[0]?.play).toMatchObject({ + bake: { file, gain: 0.5, playback: bakePlayback(spans, true) }, + volume: 0.8, + fadeIn: 2, + }); + // The second track panned the other way: the song sounds otherwise, its notes are played. + const tracks = song.tracks.map((track, i) => (i === 1 ? { ...track, pan: 1 } : track)); + f.stack.execute(new SetSongCommand(f.songId, { tracks })); + const changed = recordingAudio(); + startWorld(f, { behaviors: true, audio: changed }).frame(); + expect(changed.sequences[0]?.play.bake).toBeUndefined(); + }); + it('hands the song to the player: its tempo, intro then loop, its tracks and the level of the whole', () => { const f = playingSong(); const audio = recordingAudio(); diff --git a/tests/viewport/sound-library.test.ts b/tests/viewport/sound-library.test.ts index 7ab59016..7b63a13b 100644 --- a/tests/viewport/sound-library.test.ts +++ b/tests/viewport/sound-library.test.ts @@ -16,7 +16,7 @@ import { type FxVoice, } from '../../src/core/soundfx/nodes'; import type { MixBus } from '../../src/core/music/mixer'; -import type { SequencePlay } from '../../src/core/music/playbacks'; +import type { BakePlay, SequencePlay } from '../../src/core/music/playbacks'; import type { ScheduledNote } from '../../src/core/music/Scheduler'; import type { LfoOptions, PitchOptions } from '../../src/viewport/fxWorklet'; import { saturationCurve } from '../../src/core/soundfx/saturation'; @@ -76,9 +76,13 @@ interface FakeSource { /** The second of the audio clock it was started at, and how far into its sound (#334). */ startedAt: number; startedOffset: number; + /** How long it was told to play, where it was told (#340). */ + startedFor?: number; + loopStart?: number; + loopEnd?: number; stopped: boolean; connect: (target: unknown) => unknown; - start: (at?: number, offset?: number) => void; + start: (at?: number, offset?: number, duration?: number) => void; stop: () => void; } @@ -233,10 +237,11 @@ function fakeContext() { startedAt: -1, startedOffset: 0, stopped: false, - start: (at?: number, offset?: number) => { + start: (at?: number, offset?: number, duration?: number) => { source.started = true; source.startedAt = at ?? 0; source.startedOffset = offset ?? 0; + if (duration !== undefined) source.startedFor = duration; }, stop: () => { source.stopped = true; @@ -1898,6 +1903,86 @@ describe('the patterns and songs a game plays (#336)', () => { }); }); +describe('a bake (#340)', () => { + const BAKED = 'baked.ogg'; + /** A pattern of one note, baked: `round` null plays it once to `end`. */ + function baked( + playback: BakePlay['playback'], + strike: (note: ScheduledNote) => number = () => 0, + ): SequencePlay { + return { + sequence: { notes: [{ tick: 0, length: 480, velocity: 1, instrument: 'i', note: 60 }] }, + tempo: 120, + length: 1920, + loop: playback.round ? { start: 0, end: 1920 } : null, + needs: { layers: [], files: [] }, + strike, + bake: { file: BAKED, gain: 0.5, playback }, + }; + } + + it('is played in place of the notes: one source round its round, on the master of the playing', async () => { + const library = new SoundLibrary(false); + const struck: ScheduledNote[] = []; + const id = library.playSequence( + 'pattern:p', + baked({ end: null, until: 2, round: { start: 2, end: 4 } }, (note) => (struck.push(note), 0)), + '', + ); + await settle(); + await arrive(); + const [source] = sources; + expect(struck).toEqual([]); + expect(sources).toHaveLength(1); + expect([source!.loop, source!.loopStart, source!.loopEnd]).toEqual([true, 2, 4]); + expect([source!.startedAt, source!.startedOffset, source!.startedFor]).toEqual([0.05, 0, undefined]); + expect(library.playbacks().map((p) => [p.id, p.baked, p.planned.length])).toEqual([[id, true, 0]]); + expect(library.stats().last).toMatchObject({ file: BAKED, volume: 0.5, loop: true, bus: `play:${id}` }); + }); + + it('a head longer than the loop end hands over to a second source at the loop end, to the sample', async () => { + const library = new SoundLibrary(false); + library.playSequence('song:s', baked({ end: null, until: 2, round: { start: 3, end: 5 } }), ''); + await settle(); + await arrive(); + expect(sources.map((s) => [s.loop, s.startedAt, s.startedOffset, s.startedFor])).toEqual([ + [false, 0.05, 0, 2], + [true, 2.05, 3, undefined], + ]); + expect([sources[1]!.loopStart, sources[1]!.loopEnd]).toEqual([3, 5]); + }); + + it('played once, it stops at its end and fades out just before; let ring, it fades over 0.3 s', async () => { + const library = new SoundLibrary(false); + library.playSequence('pattern:p', baked({ end: 2, until: 2, round: null }), ''); + await settle(); + await arrive(); + expect([sources[0]!.loop, sources[0]!.startedFor]).toEqual([false, 2]); + const level = gains.at(-1)!; + expect(level.ramps).toEqual([ + ['set', 0.5, 2.04], + ['linear', 0, 2.05], + ]); + library.stopSequence('pattern:p', true); + expect(level.ramps.at(-1)).toEqual(['linear', 0, 0.3]); + }); + + it('one whose file could not be had is no bake: the notes are played', async () => { + vi.stubGlobal('fetch', () => Promise.resolve({ ok: false, status: 404 })); + const library = new SoundLibrary(false); + await library.load(BAKED); + const struck: ScheduledNote[] = []; + library.playSequence( + 'pattern:p', + baked({ end: 2, until: 2, round: null }, (note) => (struck.push(note), 0)), + '', + ); + await settle(); + expect(struck.map((note) => note.note)).toEqual([60]); + expect(library.playbacks()[0]?.baked).toBe(false); + }); +}); + describe('a sound FX on a bus (#346)', () => { /** One frame of the shifter at the fake context's 48 kHz. */ const LATE = 2048 / 48000; diff --git a/tools/e2e/64-pattern.mjs b/tools/e2e/64-pattern.mjs index c9ea1052..7c721888 100644 --- a/tools/e2e/64-pattern.mjs +++ b/tools/e2e/64-pattern.mjs @@ -570,6 +570,115 @@ export async function pattern(t) { once.length === 1 && gone.length === 0 && once[0].name === `pattern:${id}` && onceStats.playing === 0, JSON.stringify({ once: once.map((g) => g.planned.length), gone, onceStats }), ); + + // 64l'. Bake (#340) works the pattern out once, as Opus in Ogg on the pattern, and the game plays + // that while nothing it is made of has changed. A note just before the loop end rings on + // over it, which only the second round of the bake has in it. + await mutate(` + const p = __gm.state.project.assets.find((a) => a.id === ${JSON.stringify(id)}); + p.notes.push({ id: 'pt64-n4', tick: 1880, length: 40, note: 67, velocity: 1 }); + __gm.state.project.assets.find((a) => a.id === 'pt64-logic').graph.nodes.find((n) => n.id === 'pt64-play').params.loop = true;`); + await showTab('pattern'); + const bakeInfo = async () => ({ + ...(await json(`__gm.bake.info(${JSON.stringify(id)})`)), + info: await json('__gm.pattern.info()'), + }); + const unbaked = await bakeInfo(); + const patternName = await json( + `__gm.state.project.assets.find((a) => a.id === ${JSON.stringify(id)}).name`, + ); + await click(unbaked.button.x, unbaked.button.y); + const bakedNow = await waitFor(bakeInfo, (info) => info.status === 'baked', 100); + const bake = bakedNow.bake; + const listed = (await evaluate(`fetch('/game-maker/api/index.php?r=sounds').then((r) => r.json())`)).find( + (item) => item.file === bake?.file, + ); + check( + 'Bake keeps an Ogg of the pattern on it: its bar of 1.6 s, then one round more, uploaded under a marked name', + unbaked.status === 'none' && + !unbaked.button.toggled && + bakedNow.valid && + bakedNow.button.toggled && + bakedNow.info === '5 notes · Lead 64 · baked' && + /\.ogg$/.test(bake?.file ?? '') && + bake.length === 76800 && + bake.loopEnd === 76800 && + bake.head === 76800 && + bake.round === 76800 && + bake.peak > 0 && + listed?.name === `[bake] ${patternName}.ogg`, + JSON.stringify({ unbaked, bakedNow, listed, patternName }), + ); + await shot('64-baked'); + // 20 ms at a time, offline, from the notes and from the bake: within a decibel wherever it is heard. + // Silence comes over as null (-Infinity has no JSON). + const compared = await evaluate(`__gm.bake.compare(${JSON.stringify(id)}, true, 4, 0.02)`); + const audible = (compared?.live ?? []).flatMap((live, i) => + live !== null && live > -50 ? [Math.abs(live - (compared.baked[i] ?? -Infinity))] : [], + ); + const worst = Math.max(...audible); + check( + 'the bake sounds as the notes do, 20 ms at a time over two and a half rounds, the loop ends among them', + audible.length > 20 && worst < 1, + JSON.stringify({ windows: compared?.live.length, audible: audible.length, worst: +worst.toFixed(2) }), + ); + await evaluate('__gm.sound.stopAll()'); + await play(); + const bakedGame = await waitFor(patterns, (list) => list[0]?.playing, 40); + await sleep(600); + const bakedLater = { patterns: await patterns(), stats: await stats() }; + await stop(); + check( + 'the game plays the bake round and round: one voice of its file, no note planned', + bakedGame[0]?.baked && + bakedLater.patterns.length === 1 && + bakedLater.patterns[0].baked && + bakedLater.patterns[0].planned.length === 0 && + bakedLater.stats.last?.file === bake?.file && + bakedLater.stats.last?.loop === true, + JSON.stringify(bakedLater), + ); + await showTab('pattern'); + p = await json('__gm.pattern.gridAt(960, 69)'); + await click(p.x, p.y); + const outdated = await bakeInfo(); + await play(); + const liveGame = await waitFor(patterns, (list) => list[0]?.planned.length >= 2, 40); + await stop(); + check( + 'a note drawn outdates it, and the game plays the notes again', + outdated.status === 'outdated' && + !outdated.button.toggled && + outdated.info === '6 notes · Lead 64 · bake outdated' && + !outdated.valid && + liveGame[0]?.baked === false && + liveGame[0].planned.length >= 2, + JSON.stringify({ outdated, live: liveGame.map((g) => ({ baked: g.baked, planned: g.planned.length })) }), + ); + await showTab('pattern'); + await press('z', { ctrl: true }); + const undone = await bakeInfo(); + check( + 'Ctrl+Z takes the note back, and the bake holds again', + undone.status === 'baked' && undone.valid && undone.bake?.file === bake?.file, + JSON.stringify(undone), + ); + await click(undone.button.x, undone.button.y); + await clickText('Remove bake'); + const removed = await bakeInfo(); + const fileGone = await waitFor( + async () => + (await evaluate(`fetch('/game-maker/api/index.php?r=sounds').then((r) => r.json())`)).some( + (item) => item.file === bake?.file, + ), + (there) => !there, + 40, + ); + check( + 'Remove bake takes it off the pattern, and its file goes once the project is saved without it', + removed.bake === null && removed.status === 'none' && !removed.button.toggled && fileGone === false, + JSON.stringify({ removed, fileGone }), + ); const left = await rig.remove(); const idleLeft = await idle.remove();