feuerwerk/src/viewport/bakeRender.ts

/**
 * Bakes a pattern or a song (#340): plays it into an offline context with a library of its own –
 * the same code a game hears it through, its scheduler woken by the offline clock – takes what
 * reaches its master, and encodes that as Opus in Ogg with WebCodecs. Chrome has all of it.
 */
import {
  assembleBake,
  BAKE_RATE,
  type Bake,
  type BakedAsset,
  bakePrint,
  bakeRuns,
  type BakeSource,
  bakeSourceOf,
  bakeSpans,
  normalizeBake,
} from '../core/music/bake';
import { sequenceNeeds } from '../core/music/needs';
import { oggOpus, OPUS_RATE, type OpusPacket, preSkipOf } from '../core/music/ogg';
import type { BakePlay } from '../core/music/playbacks';
import type { Project } from '../core/model/Project';
import { playClip, strikeNote } from '../runtime/systems/behaviorContext';
import { SoundLibrary, soundLibrary } from './SoundLibrary';

/** Ten render quanta between two plans: about as often as the page plans (`INTERVAL`). */
const STEP = 1280;
/** Seconds of room before the first note is heard: the lead of a start and how late its buses are. */
const ROOM = 2;
/** Frames handed to the encoder at once. */
const BLOCK = 48000;
/** What libopus runs ahead at 48 kHz, where the encoder does not say. */
const DEFAULT_PRE_SKIP = 312;
const STEREO_BITRATE = 160_000;
const MONO_BITRATE = 96_000;

export class BakeCancelled extends Error {
  constructor() {
    super('Baking was cancelled');
  }
}

/** Says how far it has got, 0 to 1, and whether it is to give up. */
export interface BakeWatch {
  progress(share: number): void;
  cancelled(): boolean;
}

/** A bake as it is made, before its file has a name. */
export type MadeBake = Omit<Bake, 'file' | 'bytes'> & { data: Uint8Array };

/** The file of a bake and what goes with it; throws `BakeCancelled`, or why it cannot be made. */
export async function makeBake(project: Project, asset: BakedAsset, watch: BakeWatch): Promise<MadeBake> {
  const source = bakeSourceOf(project, asset);
  if (!source) throw new Error(`${asset.name} plays nothing, so there is nothing to bake`);
  const print = bakePrint(project, asset);
  const spans = bakeSpans(source);
  const runs = bakeRuns(spans);
  const total = runs.looped + (runs.once ?? 0);
  const share = (from: number, frames: number) => (part: number) =>
    watch.progress((0.85 * (from + part * frames)) / total);
  const looped = await renderRun(project, source, true, runs.looped, null, {
    ...watch,
    progress: share(0, runs.looped),
  });
  const once =
    runs.once === null
      ? null
      : await renderRun(project, source, false, runs.once, null, {
          ...watch,
          progress: share(runs.looped, runs.once),
        });
  // The master hears mono only where nothing on the way made it stereo; then the bake is mono too.
  const mono = looped.mono && (once?.mono ?? true);
  const sides = (run: Rendered) => (mono ? run.channels.slice(0, 1) : run.channels);
  const channels = assembleBake(spans, sides(looped), once && sides(once));
  const peak = normalizeBake(channels);
  const data = await encodeOpus(channels, {
    ...watch,
    progress: (part) => watch.progress(0.85 + 0.15 * part),
  });
  return { print, peak, ...spans, data };
}

export interface Rendered {
  channels: Float32Array[];
  /** Whether what reached the master was mono all along. */
  mono: boolean;
}

/**
 * `frames` of the source as they reach its master, from the moment its first note is heard: played
 * round its loop or once, from its notes – or from `bake` where one is given (what a check holds
 * against the notes).
 */
export async function renderRun(
  project: Project,
  source: BakeSource,
  loop: boolean,
  frames: number,
  bake: BakePlay | null,
  watch: BakeWatch,
): Promise<Rendered> {
  const context = new OfflineAudioContext(2, frames + ROOM * BAKE_RATE, BAKE_RATE);
  const pumps = new Set<() => void>();
  const library = new SoundLibrary(false, {
    context,
    files: soundLibrary,
    wake: (pump) => {
      pumps.add(pump);
      return () => pumps.delete(pump);
    },
  });
  const sink = { project, audio: library };
  const id = library.playSequence('bake', {
    sequence: source.sequence,
    tempo: source.tempo,
    length: source.length,
    loop: loop ? source.loop : null,
    needs: sequenceNeeds(project, source.sequence),
    buses: source.buses,
    strike: (note) => strikeNote(sink, note, ''),
    strikeClip: (clip) => playClip(sink, clip),
    ...(bake && { bake }),
  });
  const capture = context.createGain();
  capture.connect(context.destination);
  library.tapPlaying(id, capture);
  let most = 2;
  if (await library.worklet()) {
    most = 0;
    const probe = new AudioWorkletNode(context, 'channel-probe', { numberOfOutputs: 0 });
    probe.port.onmessage = (event: MessageEvent<number>) => {
      most = Math.max(most, event.data);
    };
    capture.connect(probe);
  }
  const anchor = await startOf(library, watch);
  const offset = Math.round(anchor * BAKE_RATE);
  if (offset + frames > context.length) throw new Error('Its buses hold it back too far to bake');
  let frame = 0;
  const hop = () => {
    frame += STEP;
    if (frame >= context.length) return;
    void context.suspend(frame / BAKE_RATE).then(() => {
      if (!watch.cancelled()) for (const pump of pumps) pump();
      watch.progress(Math.min(1, frame / (offset + frames)));
      hop();
      void context.resume();
    });
  };
  hop();
  const rendered = await context.startRendering();
  // What the probe posted last is on its way still.
  await new Promise((resolve) => setTimeout(resolve, 0));
  if (watch.cancelled()) throw new BakeCancelled();
  return {
    channels: [0, 1].map((c) => rendered.getChannelData(c).subarray(offset, offset + frames)),
    mono: most <= 1,
  };
}

/** The second of the offline clock its first note is heard at, once what it needs is loaded. */
async function startOf(library: SoundLibrary, watch: BakeWatch): Promise<number> {
  for (;;) {
    const started = library.playbacks()[0]?.startedAt;
    if (started !== undefined && started !== null) return started;
    if (watch.cancelled()) throw new BakeCancelled();
    await new Promise((resolve) => setTimeout(resolve, 10));
  }
}

/** Opus in Ogg of these samples at `OPUS_RATE`, one or two channels. */
export async function encodeOpus(channels: readonly Float32Array[], watch: BakeWatch): Promise<Uint8Array> {
  const config: AudioEncoderConfig = {
    codec: 'opus',
    sampleRate: OPUS_RATE,
    numberOfChannels: channels.length,
    bitrate: channels.length > 1 ? STEREO_BITRATE : MONO_BITRATE,
  };
  const supported =
    typeof AudioEncoder !== 'undefined' &&
    (await AudioEncoder.isConfigSupported(config).then(
      (support) => support.supported === true,
      () => false,
    ));
  if (!supported) throw new Error('This browser cannot encode Opus – baking needs Chrome');
  const packets: OpusPacket[] = [];
  // Set from the encoder's callbacks, which the compiler cannot follow into.
  const heard: { preSkip: number | null; failure: Error | null } = { preSkip: null, failure: null };
  const encoder = new AudioEncoder({
    output: (chunk, meta) => {
      const head = meta?.decoderConfig?.description;
      if (head && heard.preSkip === null) heard.preSkip = preSkipOf(bytesOf(head));
      const data = new Uint8Array(chunk.byteLength);
      chunk.copyTo(data);
      packets.push({ data, samples: Math.round(((chunk.duration ?? 0) * OPUS_RATE) / 1e6) });
    },
    error: (error) => {
      heard.failure = error;
    },
  });
  encoder.configure(config);
  const frames = channels[0]?.length ?? 0;
  for (let at = 0; at < frames && heard.failure === null && !watch.cancelled(); at += BLOCK) {
    const count = Math.min(BLOCK, frames - at);
    const planar = new Float32Array(count * channels.length);
    channels.forEach((side, c) => planar.set(side.subarray(at, at + count), c * count));
    const data = new AudioData({
      format: 'f32-planar',
      sampleRate: OPUS_RATE,
      numberOfFrames: count,
      numberOfChannels: channels.length,
      timestamp: Math.round((at * 1e6) / OPUS_RATE),
      data: planar,
    });
    encoder.encode(data);
    data.close();
    while (encoder.encodeQueueSize > 4) await new Promise((resolve) => setTimeout(resolve, 0));
    watch.progress(at / frames);
  }
  if (heard.failure === null && !watch.cancelled()) await encoder.flush();
  encoder.close();
  if (heard.failure !== null) throw heard.failure;
  if (watch.cancelled()) throw new BakeCancelled();
  return oggOpus({ channels: channels.length, preSkip: heard.preSkip ?? DEFAULT_PRE_SKIP, frames, packets });
}

function bytesOf(source: AllowSharedBufferSource): Uint8Array {
  return ArrayBuffer.isView(source)
    ? new Uint8Array(source.buffer, source.byteOffset, source.byteLength)
    : new Uint8Array(source);
}