feuerwerk/tests/core/composition-audio.test.ts

/**
 * Sounds, patterns and songs as items of a composition (#418): kept without an object, laid out as
 * sequences in milliseconds on buses of their own, and heard in a game while the composition's
 * clock runs.
 */
import { describe, expect, it } from 'vitest';
import {
  CreateAssetCommand,
  CreateSoundCommand,
  SetAssetPropertyCommand,
} from '../../src/core/commands/assetCommands';
import { SetCompositionCommand } from '../../src/core/commands/compositionCommands';
import { SetLoopCommand, SetPatternNotesCommand } from '../../src/core/commands/patternCommands';
import { PlaceAssetCommand } from '../../src/core/commands/sceneObjectCommands';
import { SetSongCommand } from '../../src/core/commands/songCommands';
import type {
  CompositionAsset,
  CompositionItem,
  InstrumentAsset,
  SongAsset,
} from '../../src/core/model/Asset';
import {
  compositionLength,
  fittedLength,
  itemFades,
  setSoundMeasure,
} from '../../src/core/model/composition';
import { carriedClips, itemMedia, mediaMarks } from '../../src/core/model/compositionEdit';
import { findAssetOf } from '../../src/core/model/Project';
import { audioItemBus, compositionPlayings, compositionSpan } from '../../src/core/music/compositionAudio';
import { MASTER_MIX, mixBusName, mixOf } from '../../src/core/music/mixer';
import { Playbacks, type SequencePlay } from '../../src/core/music/playbacks';
import { LEAD } from '../../src/core/music/Scheduler';
import { newItem } from '../../src/core/music/song';
import { MASTER_BUS, TO_COMPOSITION_BUS } from '../../src/core/properties/buses';
import { SLIP } from '../../src/runtime/compositionSound';
import { SILENT_AUDIO } from '../../src/runtime/systems/Audio';
import { addGameSystems } from '../../src/runtime/systems/BehaviorSystem';
import { Input } from '../../src/runtime/systems/Input';
import { World } from '../../src/runtime/World';
import { graphEditor, makeBehavior, makeProject } from '../helpers';

const FILE = `${'e5'.repeat(32)}.wav`;
const Q = 480;

/**
 * A composition "Cut" at 10 fps placed in a scene, a sound "Blip", a one-bar pattern "Riff" at
 * 120 BPM (two seconds) with notes on beats 1 and 3, and a song "Theme" playing it from its start.
 */
function makeCut() {
  const f = makeProject();
  const scene = new CreateAssetCommand('scene', 'Level');
  const comp = new CreateAssetCommand('composition', 'Cut', { fps: 10 });
  const sound = new CreateSoundCommand('Blip', FILE);
  const ins = new CreateAssetCommand('instrument', 'Lead');
  const pattern = new CreateAssetCommand('pattern', 'Riff');
  const song = new CreateAssetCommand('song', 'Theme');
  const bus = new CreateAssetCommand('bus', 'Music');
  for (const c of [scene, comp, sound, ins, pattern, song, bus]) f.stack.execute(c);
  (ins.asset as InstrumentAsset).graph.nodes.find((n) => n.type === 'InstrumentSample')!.params.sound =
    sound.asset.id;
  f.stack.execute(new SetAssetPropertyCommand(pattern.asset.id, 'instrument', ins.asset.id));
  f.stack.execute(new SetAssetPropertyCommand(pattern.asset.id, 'bars', 1));
  f.stack.execute(
    new SetPatternNotesCommand(pattern.asset.id, [
      { id: 'n1', tick: 0, length: Q, note: 60, velocity: 1 },
      { id: 'n2', tick: 2 * Q, length: Q, note: 64, velocity: 1 },
    ]),
  );
  const theme = () => findAssetOf(f.project, song.asset.id, 'song') as SongAsset;
  const track = theme().tracks[0]!;
  f.stack.execute(
    new SetSongCommand(song.asset.id, {
      items: [newItem({ track: track.id, kind: 'pattern', asset: pattern.asset.id, tick: 0, length: 4 * Q })],
    }),
  );
  const place = new PlaceAssetCommand(scene.asset.id, comp.asset.id, { x: 0, y: 0 });
  f.stack.execute(place);
  const cut = () => findAssetOf(f.project, comp.asset.id, 'composition') as CompositionAsset;
  const lay = (...items: CompositionItem[]) =>
    f.stack.execute(new SetCompositionCommand(cut().id, { items }));
  const audioItem = (
    id: string,
    kind: 'sound' | 'pattern' | 'song',
    start: number,
    length: number,
    more: Partial<CompositionItem> & { volume?: number; bus?: string } = {},
  ): CompositionItem => ({
    id,
    track: cut().tracks[0]!.id,
    start,
    length,
    offset: more.offset ?? 0,
    audio: {
      kind,
      asset: { sound: sound.asset.id, pattern: pattern.asset.id, song: song.asset.id }[kind],
      volume: more.volume ?? 0,
      bus: more.bus ?? '',
    },
  });
  return {
    ...f,
    sceneId: scene.asset.id,
    placement: place.objectId,
    soundId: sound.asset.id,
    patternId: pattern.asset.id,
    songId: song.asset.id,
    busId: bus.asset.id,
    cut,
    lay,
    audioItem,
  };
}

describe('audio items of a composition', () => {
  it('are kept without an object, and count to its length', () => {
    const f = makeCut();
    f.lay(f.audioItem('a', 'sound', 5, 20, { volume: 40 }), f.audioItem('b', 'pattern', 0, 10));
    expect(f.cut().items.map((i) => i.id)).toEqual(['a', 'b']);
    // Held to the most a sound item may be turned up by.
    expect(f.cut().items[0]!.audio).toEqual({ kind: 'sound', asset: f.soundId, volume: 12, bus: '' });
    expect(f.cut().objects).toEqual([]);
    expect(compositionLength(f.project, f.cut())).toBe(25);
  });

  it('drops one that plays nothing or an unknown kind', () => {
    const f = makeCut();
    const bad = {
      ...f.audioItem('a', 'sound', 0, 10),
      audio: { kind: 'video', asset: 'x', volume: 0, bus: '' },
    };
    const empty = {
      ...f.audioItem('b', 'sound', 0, 10),
      audio: { kind: 'sound', asset: '', volume: 0, bus: '' },
    };
    f.lay(bad as CompositionItem, empty as CompositionItem);
    expect(f.cut().items).toEqual([]);
  });

  it('take the length of a pattern or song, of a sound once it is measured', () => {
    const f = makeCut();
    f.lay(f.audioItem('p', 'pattern', 0, 1, { offset: 0.5 }), f.audioItem('s', 'sound', 0, 1));
    const [pattern, sound] = f.cut().items;
    expect(fittedLength(f.project, f.cut(), pattern!)).toBe(15);
    expect(fittedLength(f.project, f.cut(), sound!)).toBe(30);
    setSoundMeasure((file) => (file === FILE ? 1.25 : null));
    try {
      expect(fittedLength(f.project, f.cut(), sound!)).toBe(13);
      expect(itemMedia(f.project, f.cut(), sound!)).toEqual({ begins: 0, length: 12.5, loops: true });
    } finally {
      setSoundMeasure(() => null);
    }
  });

  it('mark a song to the end of its loop, then every time round it', () => {
    const f = makeCut();
    f.stack.execute(new SetLoopCommand('song', f.songId, { start: Q, end: 3 * Q }));
    f.lay(f.audioItem('s', 'song', 0, 40));
    const song = f.cut().items[0]!;
    expect(fittedLength(f.project, f.cut(), song)).toBe(15);
    const media = itemMedia(f.project, f.cut(), song)!;
    expect(media).toEqual({ begins: 0, length: 15, loops: true, round: 10 });
    expect(mediaMarks(song, media).seams).toEqual([15, 25, 35]);
  });

  it('own no clips: a clip on one belongs to the object item it starts on', () => {
    const f = makeCut();
    const sound = f.audioItem('a', 'sound', 0, 20);
    const object: CompositionItem = { id: 'o', track: sound.track, start: 0, length: 20, offset: 0 };
    const clip = { id: 'c', track: sound.track, animation: 'x', start: 5 };
    expect([...carriedClips([object, sound], [clip], new Set(['a']))]).toEqual([]);
    expect([...carriedClips([object, sound], [clip], new Set(['o']))]).toEqual(['c']);
  });
});

describe('the sequences of a composition', () => {
  it('lay a sound in milliseconds on a bus of its own at its volume, late by an offset below 0', () => {
    const f = makeCut();
    f.lay(
      f.audioItem('a', 'sound', 5, 20, { volume: -6, offset: 0.25 }),
      f.audioItem('b', 'sound', 30, 10, { offset: -0.3 }),
    );
    const [playing] = compositionPlayings(f.project, f.cut());
    expect(playing!.route).toBe(MASTER_MIX);
    expect(playing!.sequence.clips).toEqual([
      {
        tick: 500,
        length: 2000,
        sound: f.soundId,
        fx: '',
        offset: 0.25,
        volume: 1,
        bus: audioItemBus(f.cut().id, 'a'),
      },
      {
        tick: 3300,
        length: 700,
        sound: f.soundId,
        fx: '',
        offset: 0,
        volume: 1,
        bus: audioItemBus(f.cut().id, 'b'),
      },
    ]);
    expect(playing!.buses.map((b) => [b.name, Number(b.gain.toFixed(3))])).toEqual([
      [audioItemBus(f.cut().id, 'a'), 0.501],
      [audioItemBus(f.cut().id, 'b'), 1],
    ]);
  });

  it('fade an item on its bus: up from silence – at once without a fade in –, down at its end (#418)', () => {
    const f = makeCut();
    f.lay(
      { ...f.audioItem('a', 'sound', 5, 20, { volume: -6 }), fadeIn: 5, fadeOut: 10, curveOut: 'sCurve' },
      { ...f.audioItem('b', 'pattern', 30, 10), fadeOut: 4 },
      f.audioItem('c', 'sound', 50, 10),
    );
    const [playing] = compositionPlayings(f.project, f.cut());
    const bus = (id: string) => audioItemBus(f.cut().id, id);
    const levels = playing!.sequence.levels!.map((l) => ({ ...l, gain: Number(l.gain.toFixed(3)) }));
    expect(levels).toEqual([
      { tick: 500, length: 500, bus: bus('a'), gain: 0.501, rise: true, curve: 'linear' },
      { tick: 1500, length: 1000, bus: bus('a'), gain: 0.501, rise: false, curve: 'sCurve' },
      { tick: 3000, length: 0, bus: bus('b'), gain: 1, rise: true, curve: 'linear' },
      { tick: 3600, length: 400, bus: bus('b'), gain: 1, rise: false, curve: 'linear' },
    ]);
  });

  it('keep fades in whole frames above 0 and curves there are, and work them no longer than the item', () => {
    const f = makeCut();
    const raw = { fadeIn: 2.6, fadeOut: -3, curveIn: 'bogus', curveOut: 'equalPower' };
    f.lay({ ...f.audioItem('a', 'sound', 0, 10), ...raw } as unknown as CompositionItem);
    const [held] = f.cut().items;
    expect([held!.fadeIn, held!.fadeOut, held!.curveIn, held!.curveOut]).toEqual([
      3,
      undefined,
      undefined,
      'equalPower',
    ]);
    expect(itemFades({ ...held!, fadeIn: 50, fadeOut: 4 })).toEqual({
      fadeIn: 10,
      fadeOut: 0,
      curveIn: 'linear',
      curveOut: 'equalPower',
    });
    expect(itemFades({ ...held!, fadeIn: 6, fadeOut: 9 }).fadeOut).toBe(4);
  });

  it('play a pattern round and round at its own tempo, cut where the item ends', () => {
    const f = makeCut();
    // 3.5 s from half a second in: beat 3 (1 s), then round two (1.5 s, 2.5 s), then the next (3.5 s) is cut.
    f.lay(f.audioItem('p', 'pattern', 10, 35, { offset: 0.5 }));
    const notes = compositionPlayings(f.project, f.cut())[0]!.sequence.notes;
    expect(notes.map((n) => [n.tick, n.length, n.note, n.tempo])).toEqual([
      [1500, 500, 64, 120],
      [2500, 500, 60, 120],
      [3500, 500, 64, 120],
    ]);
  });

  it('give a song item its own copy of the song’s buses, sending into the item’s', () => {
    const f = makeCut();
    f.lay(f.audioItem('s', 'song', 0, 20), f.audioItem('t', 'song', 0, 20));
    const [playing] = compositionPlayings(f.project, f.cut());
    const bus = audioItemBus(f.cut().id, 's');
    const track =
      f.cut().tracks.length && (findAssetOf(f.project, f.songId, 'song') as SongAsset).tracks[0]!.id;
    const names = playing!.buses.map((b) => b.name);
    expect(names).toContain(`${bus}:${track}`);
    expect(names).toContain(`${audioItemBus(f.cut().id, 't')}:${track}`);
    expect(playing!.buses.find((b) => b.name === `${bus}:${track}`)!.sends).toEqual([
      expect.objectContaining({ to: bus }),
    ]);
    const own = playing!.sequence.notes.filter((n) => n.bus?.startsWith(`${bus}:`));
    expect(own.map((n) => [n.tick, n.note])).toEqual([
      [0, 60],
      [1000, 64],
    ]);
  });

  it('go one to each place of the mix their items are sent to, or all to the master unrouted', () => {
    const f = makeCut();
    f.lay(
      f.audioItem('a', 'sound', 0, 10),
      f.audioItem('b', 'sound', 0, 10, { bus: f.busId }),
      f.audioItem('c', 'pattern', 0, 10, { bus: MASTER_BUS }),
    );
    const routes = compositionPlayings(f.project, f.cut()).map((p) => [p.route, p.buses.length]);
    expect(routes).toEqual([
      [MASTER_MIX, 2],
      [mixBusName(f.busId), 1],
    ]);
    expect(compositionPlayings(f.project, f.cut(), false).map((p) => p.route)).toEqual([MASTER_MIX]);
  });

  it('go into the composition’s bus where they pick it, along their asset’s routing where they pick none (#418)', () => {
    const f = makeCut();
    f.lay(f.audioItem('a', 'sound', 0, 10), f.audioItem('b', 'pattern', 0, 10, { bus: TO_COMPOSITION_BUS }));
    const routes = compositionPlayings(f.project, f.cut()).map((p) => p.route);
    expect(routes).toEqual([MASTER_MIX, mixBusName(f.cut().id)]);
    expect(mixOf(f.project).find((bus) => bus.name === mixBusName(f.cut().id))?.into).toBe(MASTER_MIX);
  });

  it('run as long as the composition, round it where it loops', () => {
    const f = makeCut();
    f.lay(f.audioItem('a', 'sound', 5, 20));
    expect(compositionSpan(f.project, f.cut())).toEqual({ length: 2500, loop: null });
    f.stack.execute(new SetAssetPropertyCommand(f.cut().id, 'loop', true));
    expect(compositionSpan(f.project, f.cut())).toEqual({ length: 2500, loop: { start: 0, end: 2500 } });
  });
});

describe('a playing started from a tick (#418)', () => {
  it('starts as much further in as it waited for its files', async () => {
    let time = 0;
    let loaded = () => undefined as void;
    const started: number[] = [];
    const playbacks = new Playbacks({
      now: () => time,
      latency: () => 0,
      playing: () => false,
      stop: () => undefined,
      wake: () => () => undefined,
      prepare: () => new Promise<void>((resolve) => (loaded = resolve)),
      level: () => undefined,
      after: () => undefined,
    });
    const play: SequencePlay = {
      sequence: { notes: [{ tick: 0, length: 1, note: 60, velocity: 1, instrument: 'x' }] },
      tempo: 125,
      length: 10000,
      loop: null,
      needs: { layers: [], files: [] },
      from: 1000,
      strike: (note) => (started.push(note.at), 1),
    };
    playbacks.start(1, 'composition:x', play, '');
    time = 0.25;
    loaded();
    await Promise.resolve();
    const runner = playbacks.current()[0]!.runner as unknown as { position(): { ticks: number } };
    expect(runner.position().ticks).toBe(1250);
  });
});

/** A sink that writes down what it is asked to play and stop. */
function recordingAudio() {
  const sink = {
    ...SILENT_AUDIO,
    started: [] as { id: number; name: string; play: SequencePlay }[],
    stopped: [] as number[],
    /** Its audio clock, moved on by the test: 0 is none, and nothing is held to it. */
    now: 0,
    clock: () => sink.now,
    playSequence: (name: string, play: SequencePlay) => {
      const id = 100 + sink.started.length;
      sink.started.push({ id, name, play });
      return id;
    },
    stopPlayback: (id: number) => {
      sink.stopped.push(id);
    },
  };
  return sink;
}

describe('a composition played in a game', () => {
  function run(f: ReturnType<typeof makeCut>) {
    const behavior = makeBehavior(f, 'Director');
    f.stack.execute(new PlaceAssetCommand(f.sceneId, behavior.id, { x: 0, y: 0 }, f.placement));
    const edit = graphEditor(f, behavior);
    const start = edit.addNode('OnStart');
    const owner = edit.addNode('Owner');
    const play = edit.addNode('PlayComposition');
    edit.connect(start, 'exec', play, 'exec');
    edit.connect(owner, 'object', play, 'object');
    const audio = recordingAudio();
    const world = World.instantiate(f.project, f.sceneId);
    addGameSystems(world, f.project, new Input(), { audio });
    return { audio, world, frame: (times = 1) => Array.from({ length: times }, () => world.tick(0.1)) };
  }

  /** Frames of 0.1 s, the audio clock moving on by `heard` each. */
  function along(game: ReturnType<typeof run>, times: number, heard = 0.1) {
    for (let i = 0; i < times; i++) {
      game.audio.now += heard;
      game.frame();
    }
  }

  it('is heard from where its clock stands once it runs, and stops with its end', () => {
    const f = makeCut();
    f.lay(f.audioItem('a', 'sound', 0, 5));
    const game = run(f);
    game.frame();
    // The first tick shows frame 0; the clock and the sound run from the next.
    expect(game.audio.started).toEqual([]);
    game.frame();
    expect(game.audio.started).toHaveLength(1);
    expect(game.audio.started[0]!.name).toBe(`composition:${f.cut().id}`);
    expect(game.audio.started[0]!.play.from).toBe(Math.round((0.1 + LEAD) * 1000));
    game.frame(2);
    expect(game.audio.started).toHaveLength(1);
    game.frame(3);
    expect(game.audio.stopped).toEqual([100]);
  });

  it('stands silent while the game is paused, and goes on from where its clock stood', () => {
    const f = makeCut();
    f.lay(f.audioItem('a', 'sound', 0, 50));
    const game = run(f);
    game.frame(3);
    expect(game.audio.started).toHaveLength(1);
    game.world.paused = true;
    game.frame(2);
    expect(game.audio.stopped).toEqual([100]);
    expect(game.audio.started).toHaveLength(1);
    game.world.paused = false;
    game.frame();
    expect(game.audio.started).toHaveLength(2);
    expect(game.audio.started[1]!.play.from).toBe(Math.round((0.3 + LEAD) * 1000));
  });

  it('starts its sound again where the picture is once a hitch left the two apart', () => {
    const f = makeCut();
    f.lay(f.audioItem('a', 'sound', 0, 50));
    const game = run(f);
    along(game, 3);
    expect(game.audio.started).toHaveLength(1);
    // A frame of 0.6 s counts 0.1 for the picture: the sound is 0.5 s ahead.
    along(game, 1, 0.6);
    expect(game.audio.stopped).toEqual([100]);
    expect(game.audio.started).toHaveLength(2);
    expect(game.audio.started[1]!.play.from).toBe(Math.round((0.3 + LEAD) * 1000));
    // Then the two keep together, and a few milliseconds apart are no hitch.
    along(game, 3);
    along(game, 1, 0.1 + SLIP / 2);
    expect(game.audio.started).toHaveLength(2);
  });

  it('holds nothing to an audio clock that stands, nor at the turn of a loop', () => {
    const f = makeCut();
    f.lay(f.audioItem('a', 'sound', 0, 20));
    f.stack.execute(new SetAssetPropertyCommand(f.cut().id, 'loop', true));
    const game = run(f);
    along(game, 2);
    along(game, 3, 0);
    expect(game.audio.started).toHaveLength(1);
    // Round the loop of 2 s three times over, the sound with it.
    const loop = run(f);
    along(loop, 65);
    expect(loop.audio.started).toHaveLength(1);
    expect(loop.audio.stopped).toEqual([]);
  });
});