/** * Sounds, patterns and songs as items of a composition (#418): kept without an object, laid out as * sequences in milliseconds on buses of their own, and heard in a game while the composition's * clock runs. */ import { describe, expect, it } from 'vitest'; import { CreateAssetCommand, CreateSoundCommand, SetAssetPropertyCommand, } from '../../src/core/commands/assetCommands'; import { SetCompositionCommand } from '../../src/core/commands/compositionCommands'; import { SetLoopCommand, SetPatternNotesCommand } from '../../src/core/commands/patternCommands'; import { PlaceAssetCommand } from '../../src/core/commands/sceneObjectCommands'; import { SetSongCommand } from '../../src/core/commands/songCommands'; import type { CompositionAsset, CompositionItem, InstrumentAsset, SongAsset, } from '../../src/core/model/Asset'; import { compositionLength, fittedLength, itemFades, setSoundMeasure, } from '../../src/core/model/composition'; import { carriedClips, itemMedia, mediaMarks } from '../../src/core/model/compositionEdit'; import { findAssetOf } from '../../src/core/model/Project'; import { audioItemBus, compositionPlayings, compositionSpan } from '../../src/core/music/compositionAudio'; import { MASTER_MIX, mixBusName, mixOf } from '../../src/core/music/mixer'; import { Playbacks, type SequencePlay } from '../../src/core/music/playbacks'; import { LEAD } from '../../src/core/music/Scheduler'; import { newItem } from '../../src/core/music/song'; import { MASTER_BUS, TO_COMPOSITION_BUS } from '../../src/core/properties/buses'; import { SLIP } from '../../src/runtime/compositionSound'; import { SILENT_AUDIO } from '../../src/runtime/systems/Audio'; import { addGameSystems } from '../../src/runtime/systems/BehaviorSystem'; import { Input } from '../../src/runtime/systems/Input'; import { World } from '../../src/runtime/World'; import { graphEditor, makeBehavior, makeProject } from '../helpers'; const FILE = `${'e5'.repeat(32)}.wav`; const Q = 480; /** * A composition "Cut" at 10 fps placed in a scene, a sound "Blip", a one-bar pattern "Riff" at * 120 BPM (two seconds) with notes on beats 1 and 3, and a song "Theme" playing it from its start. */ function makeCut() { const f = makeProject(); const scene = new CreateAssetCommand('scene', 'Level'); const comp = new CreateAssetCommand('composition', 'Cut', { fps: 10 }); const sound = new CreateSoundCommand('Blip', FILE); const ins = new CreateAssetCommand('instrument', 'Lead'); const pattern = new CreateAssetCommand('pattern', 'Riff'); const song = new CreateAssetCommand('song', 'Theme'); const bus = new CreateAssetCommand('bus', 'Music'); for (const c of [scene, comp, sound, ins, pattern, song, bus]) f.stack.execute(c); (ins.asset as InstrumentAsset).graph.nodes.find((n) => n.type === 'InstrumentSample')!.params.sound = sound.asset.id; f.stack.execute(new SetAssetPropertyCommand(pattern.asset.id, 'instrument', ins.asset.id)); f.stack.execute(new SetAssetPropertyCommand(pattern.asset.id, 'bars', 1)); f.stack.execute( new SetPatternNotesCommand(pattern.asset.id, [ { id: 'n1', tick: 0, length: Q, note: 60, velocity: 1 }, { id: 'n2', tick: 2 * Q, length: Q, note: 64, velocity: 1 }, ]), ); const theme = () => findAssetOf(f.project, song.asset.id, 'song') as SongAsset; const track = theme().tracks[0]!; f.stack.execute( new SetSongCommand(song.asset.id, { items: [newItem({ track: track.id, kind: 'pattern', asset: pattern.asset.id, tick: 0, length: 4 * Q })], }), ); const place = new PlaceAssetCommand(scene.asset.id, comp.asset.id, { x: 0, y: 0 }); f.stack.execute(place); const cut = () => findAssetOf(f.project, comp.asset.id, 'composition') as CompositionAsset; const lay = (...items: CompositionItem[]) => f.stack.execute(new SetCompositionCommand(cut().id, { items })); const audioItem = ( id: string, kind: 'sound' | 'pattern' | 'song', start: number, length: number, more: Partial & { volume?: number; bus?: string } = {}, ): CompositionItem => ({ id, track: cut().tracks[0]!.id, start, length, offset: more.offset ?? 0, audio: { kind, asset: { sound: sound.asset.id, pattern: pattern.asset.id, song: song.asset.id }[kind], volume: more.volume ?? 0, bus: more.bus ?? '', }, }); return { ...f, sceneId: scene.asset.id, placement: place.objectId, soundId: sound.asset.id, patternId: pattern.asset.id, songId: song.asset.id, busId: bus.asset.id, cut, lay, audioItem, }; } describe('audio items of a composition', () => { it('are kept without an object, and count to its length', () => { const f = makeCut(); f.lay(f.audioItem('a', 'sound', 5, 20, { volume: 40 }), f.audioItem('b', 'pattern', 0, 10)); expect(f.cut().items.map((i) => i.id)).toEqual(['a', 'b']); // Held to the most a sound item may be turned up by. expect(f.cut().items[0]!.audio).toEqual({ kind: 'sound', asset: f.soundId, volume: 12, bus: '' }); expect(f.cut().objects).toEqual([]); expect(compositionLength(f.project, f.cut())).toBe(25); }); it('drops one that plays nothing or an unknown kind', () => { const f = makeCut(); const bad = { ...f.audioItem('a', 'sound', 0, 10), audio: { kind: 'video', asset: 'x', volume: 0, bus: '' }, }; const empty = { ...f.audioItem('b', 'sound', 0, 10), audio: { kind: 'sound', asset: '', volume: 0, bus: '' }, }; f.lay(bad as CompositionItem, empty as CompositionItem); expect(f.cut().items).toEqual([]); }); it('take the length of a pattern or song, of a sound once it is measured', () => { const f = makeCut(); f.lay(f.audioItem('p', 'pattern', 0, 1, { offset: 0.5 }), f.audioItem('s', 'sound', 0, 1)); const [pattern, sound] = f.cut().items; expect(fittedLength(f.project, f.cut(), pattern!)).toBe(15); expect(fittedLength(f.project, f.cut(), sound!)).toBe(30); setSoundMeasure((file) => (file === FILE ? 1.25 : null)); try { expect(fittedLength(f.project, f.cut(), sound!)).toBe(13); expect(itemMedia(f.project, f.cut(), sound!)).toEqual({ begins: 0, length: 12.5, loops: true }); } finally { setSoundMeasure(() => null); } }); it('mark a song to the end of its loop, then every time round it', () => { const f = makeCut(); f.stack.execute(new SetLoopCommand('song', f.songId, { start: Q, end: 3 * Q })); f.lay(f.audioItem('s', 'song', 0, 40)); const song = f.cut().items[0]!; expect(fittedLength(f.project, f.cut(), song)).toBe(15); const media = itemMedia(f.project, f.cut(), song)!; expect(media).toEqual({ begins: 0, length: 15, loops: true, round: 10 }); expect(mediaMarks(song, media).seams).toEqual([15, 25, 35]); }); it('own no clips: a clip on one belongs to the object item it starts on', () => { const f = makeCut(); const sound = f.audioItem('a', 'sound', 0, 20); const object: CompositionItem = { id: 'o', track: sound.track, start: 0, length: 20, offset: 0 }; const clip = { id: 'c', track: sound.track, animation: 'x', start: 5 }; expect([...carriedClips([object, sound], [clip], new Set(['a']))]).toEqual([]); expect([...carriedClips([object, sound], [clip], new Set(['o']))]).toEqual(['c']); }); }); describe('the sequences of a composition', () => { it('lay a sound in milliseconds on a bus of its own at its volume, late by an offset below 0', () => { const f = makeCut(); f.lay( f.audioItem('a', 'sound', 5, 20, { volume: -6, offset: 0.25 }), f.audioItem('b', 'sound', 30, 10, { offset: -0.3 }), ); const [playing] = compositionPlayings(f.project, f.cut()); expect(playing!.route).toBe(MASTER_MIX); expect(playing!.sequence.clips).toEqual([ { tick: 500, length: 2000, sound: f.soundId, fx: '', offset: 0.25, volume: 1, bus: audioItemBus(f.cut().id, 'a'), }, { tick: 3300, length: 700, sound: f.soundId, fx: '', offset: 0, volume: 1, bus: audioItemBus(f.cut().id, 'b'), }, ]); expect(playing!.buses.map((b) => [b.name, Number(b.gain.toFixed(3))])).toEqual([ [audioItemBus(f.cut().id, 'a'), 0.501], [audioItemBus(f.cut().id, 'b'), 1], ]); }); it('fade an item on its bus: up from silence – at once without a fade in –, down at its end (#418)', () => { const f = makeCut(); f.lay( { ...f.audioItem('a', 'sound', 5, 20, { volume: -6 }), fadeIn: 5, fadeOut: 10, curveOut: 'sCurve' }, { ...f.audioItem('b', 'pattern', 30, 10), fadeOut: 4 }, f.audioItem('c', 'sound', 50, 10), ); const [playing] = compositionPlayings(f.project, f.cut()); const bus = (id: string) => audioItemBus(f.cut().id, id); const levels = playing!.sequence.levels!.map((l) => ({ ...l, gain: Number(l.gain.toFixed(3)) })); expect(levels).toEqual([ { tick: 500, length: 500, bus: bus('a'), gain: 0.501, rise: true, curve: 'linear' }, { tick: 1500, length: 1000, bus: bus('a'), gain: 0.501, rise: false, curve: 'sCurve' }, { tick: 3000, length: 0, bus: bus('b'), gain: 1, rise: true, curve: 'linear' }, { tick: 3600, length: 400, bus: bus('b'), gain: 1, rise: false, curve: 'linear' }, ]); }); it('keep fades in whole frames above 0 and curves there are, and work them no longer than the item', () => { const f = makeCut(); const raw = { fadeIn: 2.6, fadeOut: -3, curveIn: 'bogus', curveOut: 'equalPower' }; f.lay({ ...f.audioItem('a', 'sound', 0, 10), ...raw } as unknown as CompositionItem); const [held] = f.cut().items; expect([held!.fadeIn, held!.fadeOut, held!.curveIn, held!.curveOut]).toEqual([ 3, undefined, undefined, 'equalPower', ]); expect(itemFades({ ...held!, fadeIn: 50, fadeOut: 4 })).toEqual({ fadeIn: 10, fadeOut: 0, curveIn: 'linear', curveOut: 'equalPower', }); expect(itemFades({ ...held!, fadeIn: 6, fadeOut: 9 }).fadeOut).toBe(4); }); it('play a pattern round and round at its own tempo, cut where the item ends', () => { const f = makeCut(); // 3.5 s from half a second in: beat 3 (1 s), then round two (1.5 s, 2.5 s), then the next (3.5 s) is cut. f.lay(f.audioItem('p', 'pattern', 10, 35, { offset: 0.5 })); const notes = compositionPlayings(f.project, f.cut())[0]!.sequence.notes; expect(notes.map((n) => [n.tick, n.length, n.note, n.tempo])).toEqual([ [1500, 500, 64, 120], [2500, 500, 60, 120], [3500, 500, 64, 120], ]); }); it('give a song item its own copy of the song’s buses, sending into the item’s', () => { const f = makeCut(); f.lay(f.audioItem('s', 'song', 0, 20), f.audioItem('t', 'song', 0, 20)); const [playing] = compositionPlayings(f.project, f.cut()); const bus = audioItemBus(f.cut().id, 's'); const track = f.cut().tracks.length && (findAssetOf(f.project, f.songId, 'song') as SongAsset).tracks[0]!.id; const names = playing!.buses.map((b) => b.name); expect(names).toContain(`${bus}:${track}`); expect(names).toContain(`${audioItemBus(f.cut().id, 't')}:${track}`); expect(playing!.buses.find((b) => b.name === `${bus}:${track}`)!.sends).toEqual([ expect.objectContaining({ to: bus }), ]); const own = playing!.sequence.notes.filter((n) => n.bus?.startsWith(`${bus}:`)); expect(own.map((n) => [n.tick, n.note])).toEqual([ [0, 60], [1000, 64], ]); }); it('go one to each place of the mix their items are sent to, or all to the master unrouted', () => { const f = makeCut(); f.lay( f.audioItem('a', 'sound', 0, 10), f.audioItem('b', 'sound', 0, 10, { bus: f.busId }), f.audioItem('c', 'pattern', 0, 10, { bus: MASTER_BUS }), ); const routes = compositionPlayings(f.project, f.cut()).map((p) => [p.route, p.buses.length]); expect(routes).toEqual([ [MASTER_MIX, 2], [mixBusName(f.busId), 1], ]); expect(compositionPlayings(f.project, f.cut(), false).map((p) => p.route)).toEqual([MASTER_MIX]); }); it('go into the composition’s bus where they pick it, along their asset’s routing where they pick none (#418)', () => { const f = makeCut(); f.lay(f.audioItem('a', 'sound', 0, 10), f.audioItem('b', 'pattern', 0, 10, { bus: TO_COMPOSITION_BUS })); const routes = compositionPlayings(f.project, f.cut()).map((p) => p.route); expect(routes).toEqual([MASTER_MIX, mixBusName(f.cut().id)]); expect(mixOf(f.project).find((bus) => bus.name === mixBusName(f.cut().id))?.into).toBe(MASTER_MIX); }); it('run as long as the composition, round it where it loops', () => { const f = makeCut(); f.lay(f.audioItem('a', 'sound', 5, 20)); expect(compositionSpan(f.project, f.cut())).toEqual({ length: 2500, loop: null }); f.stack.execute(new SetAssetPropertyCommand(f.cut().id, 'loop', true)); expect(compositionSpan(f.project, f.cut())).toEqual({ length: 2500, loop: { start: 0, end: 2500 } }); }); }); describe('a playing started from a tick (#418)', () => { it('starts as much further in as it waited for its files', async () => { let time = 0; let loaded = () => undefined as void; const started: number[] = []; const playbacks = new Playbacks({ now: () => time, latency: () => 0, playing: () => false, stop: () => undefined, wake: () => () => undefined, prepare: () => new Promise((resolve) => (loaded = resolve)), level: () => undefined, after: () => undefined, }); const play: SequencePlay = { sequence: { notes: [{ tick: 0, length: 1, note: 60, velocity: 1, instrument: 'x' }] }, tempo: 125, length: 10000, loop: null, needs: { layers: [], files: [] }, from: 1000, strike: (note) => (started.push(note.at), 1), }; playbacks.start(1, 'composition:x', play, ''); time = 0.25; loaded(); await Promise.resolve(); const runner = playbacks.current()[0]!.runner as unknown as { position(): { ticks: number } }; expect(runner.position().ticks).toBe(1250); }); }); /** A sink that writes down what it is asked to play and stop. */ function recordingAudio() { const sink = { ...SILENT_AUDIO, started: [] as { id: number; name: string; play: SequencePlay }[], stopped: [] as number[], /** Its audio clock, moved on by the test: 0 is none, and nothing is held to it. */ now: 0, clock: () => sink.now, playSequence: (name: string, play: SequencePlay) => { const id = 100 + sink.started.length; sink.started.push({ id, name, play }); return id; }, stopPlayback: (id: number) => { sink.stopped.push(id); }, }; return sink; } describe('a composition played in a game', () => { function run(f: ReturnType) { const behavior = makeBehavior(f, 'Director'); f.stack.execute(new PlaceAssetCommand(f.sceneId, behavior.id, { x: 0, y: 0 }, f.placement)); const edit = graphEditor(f, behavior); const start = edit.addNode('OnStart'); const owner = edit.addNode('Owner'); const play = edit.addNode('PlayComposition'); edit.connect(start, 'exec', play, 'exec'); edit.connect(owner, 'object', play, 'object'); const audio = recordingAudio(); const world = World.instantiate(f.project, f.sceneId); addGameSystems(world, f.project, new Input(), { audio }); return { audio, world, frame: (times = 1) => Array.from({ length: times }, () => world.tick(0.1)) }; } /** Frames of 0.1 s, the audio clock moving on by `heard` each. */ function along(game: ReturnType, times: number, heard = 0.1) { for (let i = 0; i < times; i++) { game.audio.now += heard; game.frame(); } } it('is heard from where its clock stands once it runs, and stops with its end', () => { const f = makeCut(); f.lay(f.audioItem('a', 'sound', 0, 5)); const game = run(f); game.frame(); // The first tick shows frame 0; the clock and the sound run from the next. expect(game.audio.started).toEqual([]); game.frame(); expect(game.audio.started).toHaveLength(1); expect(game.audio.started[0]!.name).toBe(`composition:${f.cut().id}`); expect(game.audio.started[0]!.play.from).toBe(Math.round((0.1 + LEAD) * 1000)); game.frame(2); expect(game.audio.started).toHaveLength(1); game.frame(3); expect(game.audio.stopped).toEqual([100]); }); it('stands silent while the game is paused, and goes on from where its clock stood', () => { const f = makeCut(); f.lay(f.audioItem('a', 'sound', 0, 50)); const game = run(f); game.frame(3); expect(game.audio.started).toHaveLength(1); game.world.paused = true; game.frame(2); expect(game.audio.stopped).toEqual([100]); expect(game.audio.started).toHaveLength(1); game.world.paused = false; game.frame(); expect(game.audio.started).toHaveLength(2); expect(game.audio.started[1]!.play.from).toBe(Math.round((0.3 + LEAD) * 1000)); }); it('starts its sound again where the picture is once a hitch left the two apart', () => { const f = makeCut(); f.lay(f.audioItem('a', 'sound', 0, 50)); const game = run(f); along(game, 3); expect(game.audio.started).toHaveLength(1); // A frame of 0.6 s counts 0.1 for the picture: the sound is 0.5 s ahead. along(game, 1, 0.6); expect(game.audio.stopped).toEqual([100]); expect(game.audio.started).toHaveLength(2); expect(game.audio.started[1]!.play.from).toBe(Math.round((0.3 + LEAD) * 1000)); // Then the two keep together, and a few milliseconds apart are no hitch. along(game, 3); along(game, 1, 0.1 + SLIP / 2); expect(game.audio.started).toHaveLength(2); }); it('holds nothing to an audio clock that stands, nor at the turn of a loop', () => { const f = makeCut(); f.lay(f.audioItem('a', 'sound', 0, 20)); f.stack.execute(new SetAssetPropertyCommand(f.cut().id, 'loop', true)); const game = run(f); along(game, 2); along(game, 3, 0); expect(game.audio.started).toHaveLength(1); // Round the loop of 2 s three times over, the sound with it. const loop = run(f); along(loop, 65); expect(loop.audio.started).toHaveLength(1); expect(loop.audio.stopped).toEqual([]); }); });