feuerwerk/src/editor/debug/soundHooks.ts

import { findAssetOf } from '../../core/model/Project';
import { projectFiles } from '../../core/model/projectFiles';
import { clipsOf, cutChannels, cutRange, findSound, parseCutName } from '../../core/model/sound';
import { STREAM_RATE } from '../../core/music/streamQueue';
import { clipCopyTable, playedPlan, streamTable } from '../../core/music/streams';
import { playedSoundOf } from '../../core/properties/playedSound';
import { soundLibrary } from '../../viewport/SoundLibrary';
import { centreOf, type HookContext } from './hookHelpers';

/** The Sound tab (#343). */
export function soundHooks({ state, editor }: HookContext) {
  const { soundEditor } = editor;
  return {
    soundTab: {
      current: () => soundEditor.currentAssetId,
      info: () => soundEditor.shownInfo,
      rect: () => soundEditor.canvasRect(),
      /** Where a second of the file is on screen, at the middle of the wave. */
      at: (seconds: number) => soundEditor.canvas.secondsToScreen(seconds),
      /** The stretch between the handles, in seconds; null while the file is not there. */
      span: () => soundEditor.canvas.shownSpan(),
      duration: () => soundEditor.canvas.fileDuration(),
      /** The height zoom (#357): the factor, the middle of the level scale, and the top of a full-scale wave. */
      level: () => soundEditor.canvas.levelZoom,
      scale: () => soundEditor.canvas.scaleCentre(),
      fullScaleTop: () => soundEditor.canvas.fullScaleTop(),
      /** The time zoom, and whether a zoom is on its way (#377). */
      zoom: () => soundEditor.canvas.zoomInfo(),
      fitButton: () => centreOf(soundEditor.fitButton),
      cursor: () => soundEditor.canvas.cursor,
      playhead: () => soundEditor.canvas.playhead,
      playing: () => soundEditor.isPlaying,
      playButton: () => ({ ...centreOf(soundEditor.playButton), text: soundEditor.playButton.text }),
      reverseButton: () => ({
        ...centreOf(soundEditor.reverseButton),
        on: soundEditor.reverseButton.toggled,
        enabled: soundEditor.reverseButton.enabled,
      }),
      /** The clips of the sound shown (#357): id, name and stored values; the picked one's id. */
      clips: () => {
        const asset = findAssetOf(state.project, soundEditor.currentAssetId, 'sound');
        return asset ? clipsOf(asset).map((c) => ({ id: c.id, name: c.name, props: { ...c.props } })) : [];
      },
      pickedClip: () => soundEditor.canvas.pickedClip()?.id ?? null,
      /** The middle of the clip strip at a second of the file, on screen. */
      stripAt: (seconds: number) => soundEditor.canvas.stripAt(seconds),
      /** The clips drawn faint over the wave, by name: all but the picked one. */
      faintClips: () => soundEditor.canvas.faintClips(),
      /** Wave draws, its shift from where it was drawn, and ruler labels given a text (#360). */
      drawInfo: () => soundEditor.canvas.drawInfo(),
      fadeInField: () => ({ ...centreOf(soundEditor.fadeInField), text: soundEditor.fadeInField.value }),
      fadeOutField: () => ({ ...centreOf(soundEditor.fadeOutField), text: soundEditor.fadeOutField.value }),
      /** The curve dropdowns show a picture of their curve; the tooltip names it. */
      curveInButton: () => ({
        ...centreOf(soundEditor.curveInButton),
        tooltip: soundEditor.curveInButton.tooltipText(),
      }),
      curveOutButton: () => ({
        ...centreOf(soundEditor.curveOutButton),
        tooltip: soundEditor.curveOutButton.tooltipText(),
      }),
      /** What a sound or a clip of one is played by: its file, or the name of the clip's cut. */
      played: (id: string) => {
        const ref = findSound(state.project, id);
        return ref ? playedSoundOf(ref) : null;
      },
      /** The samples a name plays, cut and faded: their count, rate, and the first and last few of channel 0. */
      samples: async (name: string) => {
        const buffer = await soundLibrary.load(name);
        if (!buffer) return null;
        const data = buffer.getChannelData(0);
        return {
          length: buffer.length,
          rate: buffer.sampleRate,
          head: Array.from(data.slice(0, 4)),
          tail: Array.from(data.slice(-4)),
        };
      },
    },
    /** Streamed sounds (#349). */
    streams: {
      /** Done once every copy wanted has been made. */
      idle: () => editor.streamKeeper.idle(),
      /** Every name a game streams, with its copy. */
      table: () => [...streamTable(state.project)].map(([name, copy]) => ({ name, ...copy })),
      /**
       * Streams `seconds` of a name offline from `offset` seconds in, heard from `at`, looped or not,
       * and holds it against its copy decoded whole: the largest difference of any sample from what
       * should be heard there – silence before `at` – against the feed's own decoder and against the
       * browser's `decodeAudioData`, and how many samples came too late.
       */
      exact: async (name: string, offset: number, loop: boolean, at: number, seconds: number) => {
        const entry = streamTable(state.project).get(name);
        if (!entry) return null;
        const plan = playedPlan(entry.frames, offset, loop);
        const frames = Math.round((at + seconds) * STREAM_RATE);
        const [played, whole, browser] = await Promise.all([
          soundLibrary.renderStream(name, plan, at, frames),
          soundLibrary.streamedWhole(name),
          soundLibrary.decodeApart(entry.file, STREAM_RATE),
        ]);
        if (!played || !whole) return null;
        const first = Math.round(at * STREAM_RATE);
        const placeOf = (i: number) =>
          loop ? (plan.from + i - first) % entry.frames : plan.from + i - first;
        const worst = (reference: (c: number, place: number) => number) => {
          let most = 0;
          played.channels.forEach((side, c) => {
            for (let i = 0; i < frames; i++) {
              const place = placeOf(i);
              const expected = i < first || place >= entry.frames ? 0 : reference(c, place);
              most = Math.max(most, Math.abs(side[i]! - expected));
            }
          });
          return most;
        };
        return {
          frames: entry.frames,
          decoded: browser.length,
          own: worst((c, place) => whole[c]![place]!),
          browser: worst(
            (c, place) => browser.getChannelData(Math.min(c, browser.numberOfChannels - 1))[place]!,
          ),
          skipped: played.skipped,
        };
      },
      /**
       * What a game decodes that has just opened the project, in a library of its own so nothing this
       * page holds counts: the names it decodes ahead, whether that decoded the whole file of a name's
       * sound, and what the name then plays held against its cut made from that file – samples, rate,
       * the loudest difference of a sample and of the level of any 20 ms.
       */
      fresh: async (name: string) => {
        const library = soundLibrary.aside();
        library.setStreams(streamTable(state.project));
        library.setCopies(clipCopyTable(state.project));
        const ahead = projectFiles(state.project).sounds;
        await Promise.all(ahead.map((file) => library.load(file)));
        const { file, cut } = parseCutName(name);
        const whole = library.decoded(file) !== null;
        const played = await library.load(name);
        const original = await soundLibrary.decodeApart(file, STREAM_RATE);
        if (!played || !cut || played.sampleRate !== STREAM_RATE) return { ahead, whole, played: null };
        const [reference] = cutChannels(
          [original.getChannelData(0)],
          cutRange(original.length, STREAM_RATE, cut),
        );
        const heard = played.getChannelData(0);
        let worst = 0;
        let level = 0;
        const span = Math.round(0.02 * STREAM_RATE);
        for (let from = 0; from + span <= heard.length; from += span) {
          let own = 0;
          let wanted = 0;
          for (let i = from; i < from + span; i++) {
            worst = Math.max(worst, Math.abs(heard[i]! - (reference![i] ?? 0)));
            own += heard[i]! ** 2;
            wanted += (reference![i] ?? 0) ** 2;
          }
          // Only where it is heard: the fades start from silence.
          if (wanted / span > 1e-6) level = Math.max(level, Math.abs(10 * Math.log10(own / wanted)));
        }
        return {
          ahead,
          whole,
          played: { length: played.length, wanted: reference!.length, worst, level },
          decoded: library.stats().decoded,
        };
      },
      /** What a sound and each of its clips keep of their copies. */
      copies: (id: string) => {
        const asset = findAssetOf(state.project, id, 'sound');
        if (!asset) return null;
        return {
          sound: asset.stream ?? null,
          clips: clipsOf(asset).map((clip) => ({ id: clip.id, stream: clip.stream ?? null })),
        };
      },
    },
  };
}