feuerwerk/tests/core/sounds.test.ts

/**
 * A sound is an asset holding an uploaded file. Nothing wears it and nothing places it – a graph
 * plays it, and what the asset says about volume and looping is what a bare `Play sound` does.
 */
import { describe, expect, it } from 'vitest';
import {
  buildAsset,
  CreateAssetCommand,
  CreateSoundCommand,
  SetAssetPropertyCommand,
} from '../../src/core/commands/assetCommands';
import { AddClipCommand, SetClipPropsCommand } from '../../src/core/commands/clipCommands';
import type { BehaviorAsset, SoundAsset } from '../../src/core/model/Asset';
import { isPlaceableType } from '../../src/core/model/AssetTypes';
import {
  cutChannels,
  cutName,
  cutRange,
  FADE_CURVES,
  fadeGain,
  isSoundFile,
  parseCutName,
  soundFileOf,
} from '../../src/core/model/sound';
import { assetRefChoices } from '../../src/core/properties/assetRef';
import { schemaFor } from '../../src/core/properties/PropertySchema';
import { playedSoundById, playedSoundOf } from '../../src/core/properties/playedSound';
import {
  CLIP_ROWS,
  cutOf,
  END_KEY,
  FADE_IN_CURVE_KEY,
  FADE_OUT_CURVE_KEY,
  FADE_IN_KEY,
  FADE_OUT_KEY,
  LOOP_KEY,
  loopsOf,
  REVERSE_KEY,
  START_KEY,
  VOLUME_KEY,
  volumeOf,
} from '../../src/core/properties/sounds';
import { deserializeProject } from '../../src/core/store/serialize';
import type { AudioSink } from '../../src/runtime/systems/Audio';
import {
  graphEditor,
  makeBehavior,
  makeProject,
  makeSpriteScene,
  place,
  roundTrip,
  type SceneFixture,
  startWorld,
} from '../helpers';

const FILE = `${'a1'.repeat(32)}.wav`;

/** Records what it was asked to play instead of making a noise – the AudioSink a test wants. */
function fakeAudio(): AudioSink & {
  played: { file: string; volume: number; loop: boolean; owner: string }[];
  stopped: string[];
  stoppedOwners: string[];
} {
  const sink = {
    played: [] as { file: string; volume: number; loop: boolean; owner: string }[],
    stopped: [] as string[],
    stoppedOwners: [] as string[],
    play: (file: string, volume: number, loop: boolean, owner: string) =>
      sink.played.push({ file, volume, loop, owner }),
    playNote: () => 0,
    playSequence: () => 0,
    stopSequence: () => {},
    playClip: () => 0,
    stopPlayback: () => {},
    release: () => {},
    tune: () => {},
    length: () => 0,
    tail: () => 0,
    stop: (file: string) => sink.stopped.push(file),
    stopVoice: () => {},
    stopOwner: (owner: string) => sink.stoppedOwners.push(owner),
    setMix: () => {},
    setMixVolume: () => {},
    mixVolume: () => 1,
    resetMix: () => {},
    tuneMix: () => {},
  };
  return sink;
}

function makeSound(name = 'Jump', file = FILE) {
  const fixture = makeProject();
  const command = new CreateSoundCommand(name, file);
  fixture.stack.execute(command);
  return { ...fixture, sound: command.asset };
}

/** A scene whose one behavior plays `sound` on start, with whatever the node itself carries. */
function playingScene(params: Record<string, string | number | boolean> = {}) {
  const f: SceneFixture & { behavior?: BehaviorAsset } = makeSpriteScene('Box');
  const create = new CreateSoundCommand('Jump', FILE);
  f.stack.execute(create);
  const behavior = makeBehavior(f, 'Logic');
  const box = place(f, f.spriteId, { x: 0, y: 0 });
  place(f, behavior.id, { x: 0, y: 0 }, box);
  const edit = graphEditor(f, behavior);
  const start = edit.addNode('OnStart');
  const play = edit.addNode('PlaySound', { sound: create.asset.id, volume: 1, ...params });
  edit.connect(start, 'exec', play, 'exec');
  return { ...f, sound: create.asset, behavior, play };
}

describe('sound files', () => {
  it('accepts only a content-addressed name in a supported format', () => {
    expect(isSoundFile(FILE)).toBe(true);
    expect(isSoundFile(`${'b3'.repeat(32)}.mp3`)).toBe(true);
    expect(isSoundFile(`${'b3'.repeat(32)}.m4a`)).toBe(true);
    expect(isSoundFile(`${'a1'.repeat(32)}.png`)).toBe(false);
    expect(isSoundFile(`${'a1'.repeat(32)}.WAV`)).toBe(false);
    expect(isSoundFile('jump.wav')).toBe(false);
    expect(isSoundFile('')).toBe(false);
  });
});

describe('the sound asset', () => {
  it('is not placeable, starts without a file, and is full volume and one-shot by default', () => {
    expect(isPlaceableType('sound')).toBe(false);
    const asset = buildAsset('sound', 'Sound');
    expect(asset).toMatchObject({ type: 'sound', file: '' });
    expect(volumeOf(asset.props)).toBe(1);
    expect(loopsOf(asset.props)).toBe(false);
    // The cut belongs to a clip of the sound (#357), the sound is the whole file.
    expect(schemaFor('sound').map((def) => def.key)).toEqual([VOLUME_KEY, LOOP_KEY, 'fx']);
    expect(CLIP_ROWS.map((def) => def.key)).toEqual([
      START_KEY,
      END_KEY,
      REVERSE_KEY,
      FADE_IN_KEY,
      FADE_IN_CURVE_KEY,
      FADE_OUT_KEY,
      FADE_OUT_CURVE_KEY,
    ]);
    expect(cutOf({})).toEqual({
      start: 0,
      end: 0,
      fadeIn: 0.005,
      fadeOut: 0.005,
      curveIn: 'linear',
      curveOut: 'linear',
      reverse: false,
    });
  });

  it('refuses a file name the server could not have produced', () => {
    const fixture = makeProject();
    expect(() => fixture.stack.execute(new CreateSoundCommand('Bad', 'jump.wav'))).toThrow(
      'is not a stored sound file',
    );
    expect(fixture.project.assets).toHaveLength(0);
  });

  it('survives a round trip and drops a file name it cannot trust', () => {
    const s = makeSound();
    expect(roundTrip(s.project)).toEqual(s.project);

    const repaired = deserializeProject({
      name: 'P',
      assets: [
        { id: 's1', type: 'sound', name: 'Local', file: '../../etc/passwd' },
        { id: 's2', type: 'sound', name: 'Gone' },
      ],
    });
    expect(repaired.assets.map((a) => (a as SoundAsset).file)).toEqual(['', '']);
  });

  it('resolves to a file, and to none when the id is unset, dangling or of another type', () => {
    const s = makeSound();
    expect(soundFileOf(s.project, s.sound.id)).toBe(FILE);
    expect(soundFileOf(s.project, '')).toBe('');
    expect(soundFileOf(s.project, 'no-such-asset')).toBe('');

    const text = new CreateAssetCommand('text', 'Title');
    s.stack.execute(text);
    expect(soundFileOf(s.project, text.asset.id)).toBe('');
  });

  it('is what a sound picker offers, and nothing else is', () => {
    const s = makeSound();
    s.stack.execute(new CreateAssetCommand('text', 'Title'));
    expect(assetRefChoices(s.project, ['sound']).map((a) => a.id)).toEqual([s.sound.id]);
  });
});

describe('the cut of a sound (#343)', () => {
  it('is no cut at all over the whole file, and a name of its own otherwise', () => {
    const linear = { curveIn: 'linear', curveOut: 'linear', reverse: false } as const;
    const whole = { start: 0, end: 0, fadeIn: 0.005, fadeOut: 0.005, ...linear };
    expect(cutName(FILE, whole)).toBe(FILE);
    expect(cutName('', { ...whole, start: 1 })).toBe('');
    const cut = {
      start: 0.25,
      end: 1.5,
      fadeIn: 0.005,
      fadeOut: 0.2,
      curveIn: 'sCurve',
      curveOut: 'equalPower',
      reverse: false,
    } as const;
    const name = cutName(FILE, cut);
    expect(name).toBe(`${FILE}#0.25~1.5~0.005~0.2~sCurve~equalPower`);
    expect(parseCutName(name)).toEqual({ file: FILE, cut });
    expect(parseCutName(FILE)).toEqual({ file: FILE, cut: null });
    expect(parseCutName(`${FILE}#x~-2`)).toEqual({
      file: FILE,
      cut: { start: 0, end: 0, fadeIn: 0, fadeOut: 0, ...linear },
    });
  });

  it('keeps at least a sample inside the file, and ramps only the edges inside it, each by its own fade', () => {
    // 10 samples at 10 Hz: one sample per tenth of a second.
    const fades = (fadeIn: number, fadeOut = fadeIn) => ({
      fadeIn,
      fadeOut,
      curveIn: 'linear' as const,
      curveOut: 'linear' as const,
      reverse: false,
    });
    expect(cutRange(10, 10, { start: 0.2, end: 0.8, ...fades(0.2) })).toEqual({
      from: 2,
      to: 8,
      fadeIn: 2,
      fadeOut: 2,
      curveIn: 'linear',
      curveOut: 'linear',
      reverse: false,
    });
    expect(cutRange(10, 10, { start: 0.1, end: 0.9, ...fades(0.1, 0.3) })).toMatchObject({
      fadeIn: 1,
      fadeOut: 3,
    });
    expect(cutRange(10, 10, { start: 0, end: 0.5, ...fades(0.2) })).toMatchObject({ fadeIn: 0, fadeOut: 2 });
    expect(cutRange(10, 10, { start: 0.5, end: 0, ...fades(0.2) })).toMatchObject({
      to: 10,
      fadeIn: 2,
      fadeOut: 0,
    });
    // Past the end, an end before the start, and fades longer than half of what is kept.
    expect(cutRange(10, 10, { start: 5, end: 0, ...fades(0) })).toMatchObject({ from: 9, to: 10 });
    expect(cutRange(10, 10, { start: 0.6, end: 0.3, ...fades(0) })).toMatchObject({ from: 6, to: 7 });
    expect(cutRange(10, 10, { start: 0.2, end: 0.6, ...fades(1, 0.1) })).toMatchObject({
      fadeIn: 2,
      fadeOut: 1,
    });
  });

  it('cuts every channel alike, rising from silence and falling to it at an inner edge', () => {
    const left = new Float32Array(10).fill(1);
    const right = new Float32Array(10).fill(-0.5);
    const [l, r] = cutChannels([left, right], {
      from: 2,
      to: 8,
      fadeIn: 2,
      fadeOut: 2,
      curveIn: 'linear',
      curveOut: 'linear',
      reverse: false,
    });
    expect(Array.from(l!)).toEqual([0, 0.5, 1, 1, 0.5, 0]);
    expect(Array.from(r!)).toEqual([-0, -0.25, -0.5, -0.5, -0.25, -0]);
    expect(Array.from(left)).toEqual(new Array(10).fill(1));
  });

  it('shapes each ramp by its own curve, the end running it backwards (#345)', () => {
    for (const { value } of FADE_CURVES) {
      expect(fadeGain(value, 0)).toBeCloseTo(0, 12);
      expect(fadeGain(value, 1)).toBeCloseTo(1, 12);
    }
    expect(fadeGain('linear', 0.5)).toBe(0.5);
    expect(fadeGain('equalPower', 0.5)).toBeCloseTo(Math.SQRT1_2, 12);
    expect(fadeGain('sCurve', 0.25)).toBeCloseTo(1 - fadeGain('sCurve', 0.75), 12);
    expect(fadeGain('exponential', 0.5)).toBe(0.25);
    expect(fadeGain('logarithmic', 0.5)).toBe(0.75);
    const [kept] = cutChannels([new Float32Array(12).fill(1)], {
      from: 0,
      to: 12,
      fadeIn: 4,
      fadeOut: 4,
      curveIn: 'exponential',
      curveOut: 'logarithmic',
      reverse: false,
    });
    expect(Array.from(kept!)).toEqual([0, 0.0625, 0.25, 0.5625, 1, 1, 1, 1, 0.9375, 0.75, 0.4375, 0]);
  });

  it('is what every way of playing a clip is given, its fades in milliseconds on the clip (#357)', () => {
    const f = playingScene();
    expect(playedSoundOf({ asset: f.sound, clip: null })).toBe(FILE);
    const add = new AddClipCommand(f.sound.id, { [START_KEY]: 0.5, [FADE_IN_KEY]: 25 });
    f.stack.execute(add);
    f.stack.execute(
      new SetClipPropsCommand(f.sound.id, add.clipId, {
        [FADE_OUT_KEY]: 40,
        [FADE_OUT_CURVE_KEY]: 'equalPower',
      }),
    );
    const name = `${FILE}#0.5~0~0.025~0.04~linear~equalPower`;
    expect(playedSoundById(f.project, add.clipId)).toBe(name);
    // The sound itself stays the whole file.
    expect(playedSoundById(f.project, f.sound.id)).toBe(FILE);
    expect(playedSoundById(f.project, '')).toBe('');
    f.play.params.sound = add.clipId;
    const audio = fakeAudio();
    startWorld(f, { behaviors: true, audio }).frame();
    expect(audio.played).toMatchObject([{ file: name }]);
  });

  it('plays backwards from its end, the fade in ramping there (#321)', () => {
    const fades = { fadeIn: 0.2, fadeOut: 0.2, curveIn: 'linear', curveOut: 'linear' } as const;
    // Kept up to the middle of a 10-sample file: only the end lies inside, and it is heard first.
    expect(cutRange(10, 10, { start: 0, end: 0.5, ...fades, reverse: true })).toMatchObject({
      from: 0,
      to: 5,
      fadeIn: 2,
      fadeOut: 0,
      reverse: true,
    });
    const ramp = Float32Array.from({ length: 10 }, (_, i) => i);
    const [kept] = cutChannels([ramp], {
      from: 2,
      to: 8,
      fadeIn: 2,
      fadeOut: 1,
      curveIn: 'linear',
      curveOut: 'linear',
      reverse: true,
    });
    expect(Array.from(kept!)).toEqual([0, 3, 5, 4, 3, 0]);
    expect(Array.from(ramp)).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);

    const f = playingScene();
    const add = new AddClipCommand(f.sound.id, { [REVERSE_KEY]: true });
    f.stack.execute(add);
    const name = `${FILE}#0~0~0.005~0.005~linear~linear~reverse`;
    expect(playedSoundById(f.project, add.clipId)).toBe(name);
    expect(parseCutName(name).cut).toMatchObject({ start: 0, end: 0, reverse: true });
    f.stack.execute(new SetClipPropsCommand(f.sound.id, add.clipId, { [REVERSE_KEY]: false }));
    expect(playedSoundById(f.project, add.clipId)).toBe(FILE);
  });
});

describe('Play sound', () => {
  it('plays the asset as the project describes it', () => {
    const f = playingScene();
    const audio = fakeAudio();
    startWorld(f, { behaviors: true, audio }).frame();
    expect(audio.played).toMatchObject([{ file: FILE, volume: 1, loop: false }]);
  });

  it('scales the asset volume rather than replacing it, and clamps the product', () => {
    const f = playingScene({ volume: 0.5 });
    f.stack.execute(new SetAssetPropertyCommand(f.sound.id, VOLUME_KEY, 0.4));
    const audio = fakeAudio();
    startWorld(f, { behaviors: true, audio }).frame();
    expect(audio.played[0]?.volume).toBeCloseTo(0.2);

    const loud = playingScene({ volume: 10 });
    const loudAudio = fakeAudio();
    startWorld(loud, { behaviors: true, audio: loudAudio }).frame();
    expect(loudAudio.played[0]?.volume).toBe(1);
  });

  it('loops as the asset says while the node says nothing', () => {
    const asset = playingScene();
    asset.stack.execute(new SetAssetPropertyCommand(asset.sound.id, LOOP_KEY, true));
    const assetAudio = fakeAudio();
    startWorld(asset, { behaviors: true, audio: assetAudio }).frame();
    expect(assetAudio.played[0]?.loop).toBe(true);

    const node = playingScene({ loop: 'true' });
    const nodeAudio = fakeAudio();
    startWorld(node, { behaviors: true, audio: nodeAudio }).frame();
    expect(nodeAudio.played[0]?.loop).toBe(true);
  });

  /** The reason the node asks for three answers rather than a checkbox: music played as a one-shot. */
  it('plays a looping asset once when the node says off', () => {
    const f = playingScene({ loop: 'false' });
    f.stack.execute(new SetAssetPropertyCommand(f.sound.id, LOOP_KEY, true));
    const audio = fakeAudio();
    startWorld(f, { behaviors: true, audio }).frame();
    expect(audio.played[0]?.loop).toBe(false);
  });

  it('takes a boolean arriving over an edge literally', () => {
    const on = playingScene();
    const onEdit = graphEditor(on, on.behavior);
    onEdit.connect(onEdit.addNode('Boolean', { value: true }), 'value', on.play, 'loop');
    const onAudio = fakeAudio();
    startWorld(on, { behaviors: true, audio: onAudio }).frame();
    expect(onAudio.played[0]?.loop).toBe(true);

    const off = playingScene();
    off.stack.execute(new SetAssetPropertyCommand(off.sound.id, LOOP_KEY, true));
    const offEdit = graphEditor(off, off.behavior);
    offEdit.connect(offEdit.addNode('Boolean', { value: false }), 'value', off.play, 'loop');
    const offAudio = fakeAudio();
    startWorld(off, { behaviors: true, audio: offAudio }).frame();
    expect(offAudio.played[0]?.loop).toBe(false);
  });

  it('plays nothing for an id that resolves to no sound, and does not stop the flow', () => {
    const f = playingScene();
    const edit = graphEditor(f, f.behavior);
    const log = edit.addNode('Log', { message: 'after' });
    edit.connect(f.play, 'exec', log, 'exec');
    f.play.params.sound = 'no-such-asset';

    const audio = fakeAudio();
    const { world, frame } = startWorld(f, { behaviors: true, audio });
    frame();
    expect(audio.played).toEqual([]);
    expect(world.log).toEqual(['after']);
  });
});

/**
 * A sound is something a thing in the world is *doing*, so it ends when that thing does – the
 * `Stop sound` you would otherwise have to remember in every `On destroy`.
 */
describe('a voice belongs to the behavior that started it', () => {
  it('names that behavior as its owner, and nobody when the node is detached', () => {
    const owned = playingScene();
    const ownedAudio = fakeAudio();
    const run = startWorld(owned, { behaviors: true, audio: ownedAudio });
    run.frame();
    const logic = run.world.entities.find((e) => e.name === 'Logic')!;
    expect(ownedAudio.played[0]?.owner).toBe(logic.id);

    const free = playingScene({ detached: true });
    const freeAudio = fakeAudio();
    startWorld(free, { behaviors: true, audio: freeAudio }).frame();
    expect(freeAudio.played[0]?.owner).toBe('');
  });

  it('is silenced when the behavior is destroyed, after On destroy has had its say', () => {
    const f = playingScene();
    const edit = graphEditor(f, f.behavior);
    const key = edit.addNode('OnKeyDown', { key: 'x' });
    const owner = edit.addNode('Owner');
    const destroy = edit.addNode('Destroy');
    edit.wire([
      [key, 'exec', destroy, 'exec'],
      [owner, 'object', destroy, 'object'],
    ]);

    const audio = fakeAudio();
    const { world, input, frame } = startWorld(f, { behaviors: true, audio });
    frame();
    const logic = world.entities.find((e) => e.name === 'Logic')!;
    expect(audio.stoppedOwners).toEqual([]);

    input.press('x');
    frame();
    expect(audio.stoppedOwners).toEqual([logic.id]);
  });
});

describe('Stop sound', () => {
  it('silences one sound by its file, and everything when it names none', () => {
    const f = playingScene();
    const edit = graphEditor(f, f.behavior);
    const stop = edit.addNode('StopSound', { sound: f.sound.id });
    const all = edit.addNode('StopSound', { sound: '' });
    edit.connect(f.play, 'exec', stop, 'exec');
    edit.connect(stop, 'exec', all, 'exec');

    const audio = fakeAudio();
    startWorld(f, { behaviors: true, audio }).frame();
    expect(audio.stopped).toEqual([FILE, '']);
  });

  it('silences nothing at all for an id that resolves to no sound', () => {
    const f = playingScene();
    const edit = graphEditor(f, f.behavior);
    const stop = edit.addNode('StopSound', { sound: 'no-such-asset' });
    edit.connect(f.play, 'exec', stop, 'exec');

    const audio = fakeAudio();
    startWorld(f, { behaviors: true, audio }).frame();
    // Not `['']`: a dangling reference must not turn into "stop everything".
    expect(audio.stopped).toEqual([]);
  });
});