feuerwerk/src/core/model/composition.ts

/**
 * Compositions (#417): objects along a time axis, counted in frames of the composition's own `fps`.
 * What runs at a frame is one pure function, `compositionAt`, so the tab's preview and the running
 * game read the same cut; `compositionHolder` writes it out as objects for a view to draw.
 */
import { newId } from '../ids';
import { patternLength, patternTempo } from '../music/pattern';
import { ITEM_MIN_DB, songLoop, songTempo, SOUND_ITEM_MAX_DB } from '../music/song';
import { secondsOfTicks } from '../music/time';
import {
  COMPOSITION_FPS_KEY,
  COMPOSITION_LOOP_KEY,
  DEFAULT_COMPOSITION_FPS,
} from '../properties/compositions';
import { playedSoundById } from '../properties/playedSound';
import { resolveSceneObjectProps } from '../properties/resolve';
import { DEFAULT_FADE_CURVE } from '../properties/sounds';
import {
  VIDEO_CLOCK_KEY,
  VIDEO_LEVEL_KEY,
  VIDEO_TIME_KEY,
  videoDrivenOf,
  videoLevelOf,
} from '../properties/videos';
import { animationDuration, fpsOf, frameCount, targetNamePath } from './animation';
import { type ReadAt, writeObjectRow } from './animationPreview';
import type {
  AnimationAsset,
  AnimationTrack,
  Asset,
  CompositionAsset,
  CompositionAudio,
  CompositionAudioKind,
  CompositionClip,
  CompositionItem,
  CompositionTrack,
  SpriteAsset,
} from './Asset';
import { curveLookup } from './curve';
import {
  contentId,
  instanceChildId,
  objectRef,
  placedWithContent,
  resolveObjects,
  splitInstanceId,
} from './instance';
import { findAsset, findAssetOf, type Project } from './Project';
import { booleanOf, type PropertyBag } from './Properties';
import { objectName, type SceneObject } from './SceneObject';
import { type FadeCurve, fadeGain, isFadeCurve } from './sound';
import { findVideo, videoLengthOf, type VideoRef } from './video';

/** How long an item lasts while it has no video to take its length from. */
export const DEFAULT_ITEM_SECONDS = 3;

export function compositionFps(comp: CompositionAsset): number {
  const fps = Number(comp.props[COMPOSITION_FPS_KEY]);
  return Number.isFinite(fps) && fps > 0 ? fps : DEFAULT_COMPOSITION_FPS;
}

export function compositionLoops(comp: CompositionAsset): boolean {
  return booleanOf(comp.props[COMPOSITION_LOOP_KEY]);
}

/** A sprite a composition keeps to itself (#419), worn by the items a texture or video was laid as. */
export function isCompositionSprite(asset: Asset): asset is SpriteAsset {
  return asset.type === 'sprite' && (asset.owner ?? '') !== '';
}

export function newCompositionTrack(index: number): CompositionTrack {
  return { id: newId(), name: `Track ${index}` };
}

export function itemEnd(item: CompositionItem): number {
  return item.start + item.length;
}

/** The object an item spawns; undefined once it is gone, and then the item does nothing – or for an audio item. */
export function itemObject(comp: CompositionAsset, item: CompositionItem): SceneObject | undefined {
  return item.audio ? undefined : comp.objects.find((o) => o.id === item.id);
}

/** Whether an item stands on its track: an audio item always (#418), any other while it has its object. */
export function itemStands(comp: CompositionAsset, item: CompositionItem): boolean {
  return item.audio !== undefined || itemObject(comp, item) !== undefined;
}

/** Seconds a sound file lasts once its bytes are there, as the editor's sound library knows it (#418). */
let soundSeconds: (file: string) => number | null = () => null;

export function setSoundMeasure(measure: (file: string) => number | null): void {
  soundSeconds = measure;
}

/** Whether an item has a sound its fades shape: what an audio item plays, or the video its object wears. */
export function itemSounds(project: Project, comp: CompositionAsset, item: CompositionItem): boolean {
  return item.audio !== undefined || itemVideo(project, comp, item) !== null;
}

/**
 * The fades of an item in whole frames as they work: the fade in no longer than the item, the fade
 * out no longer than the rest.
 */
export interface ItemFades {
  fadeIn: number;
  fadeOut: number;
  curveIn: FadeCurve;
  curveOut: FadeCurve;
}

export function itemFades(item: CompositionItem): ItemFades {
  const frames = (value: number | undefined, most: number) =>
    Math.max(0, Math.min(most, Math.round(Number.isFinite(value) ? value! : 0)));
  const fadeIn = frames(item.fadeIn, item.length);
  return {
    fadeIn,
    fadeOut: frames(item.fadeOut, item.length - fadeIn),
    curveIn: item.curveIn ?? DEFAULT_FADE_CURVE,
    curveOut: item.curveOut ?? DEFAULT_FADE_CURVE,
  };
}

/** How loud an item's sound is `at` frames after its start, its fades considered: 0 to 1. */
export function fadeLevel(item: CompositionItem, at: number): number {
  const { fadeIn, fadeOut, curveIn, curveOut } = itemFades(item);
  const rise = fadeIn > 0 && at < fadeIn ? fadeGain(curveIn, at / fadeIn) : 1;
  const left = item.length - at;
  const fall = fadeOut > 0 && left < fadeOut ? fadeGain(curveOut, left / fadeOut) : 1;
  return rise * fall;
}

/**
 * Seconds the first pass of what an audio item plays lasts: its sound once loaded, its pattern, its
 * song to the end of its loop – it goes round there, what lies behind is never heard –; 0 while that
 * is not known or the asset is gone.
 */
export function audioSeconds(project: Project, audio: CompositionAudio): number {
  if (audio.kind === 'sound') {
    const file = playedSoundById(project, audio.asset);
    return (file && soundSeconds(file)) || 0;
  }
  if (audio.kind === 'pattern') {
    const pattern = findAssetOf(project, audio.asset, 'pattern');
    return pattern ? secondsOfTicks(patternLength(pattern), patternTempo(pattern.props)) : 0;
  }
  const song = findAssetOf(project, audio.asset, 'song');
  return song ? secondsOfTicks(songLoop(song).end, songTempo(song.props)) : 0;
}

/** Seconds every pass after the first lasts: a song's loop, else the same as the first. */
export function audioRoundSeconds(project: Project, audio: CompositionAudio): number {
  const song = audio.kind === 'song' ? findAssetOf(project, audio.asset, 'song') : undefined;
  if (!song) return audioSeconds(project, audio);
  const loop = songLoop(song);
  return secondsOfTicks(loop.end - loop.start, songTempo(song.props));
}

/**
 * The animation a clip plays, unless it is gone or built over a scene: a scene's rows aim at that
 * scene's objects, nothing an item holds.
 */
function clipAnimation(project: Project, clip: CompositionClip): AnimationAsset | undefined {
  const animation = findAssetOf(project, clip.animation, 'animation');
  return animation && findAsset(project, animation.root)?.type !== 'scene' ? animation : undefined;
}

/** Frames one pass of the clip's animation takes in the composition; 0 when it plays nothing. */
export function clipLength(project: Project, comp: CompositionAsset, clip: CompositionClip): number {
  const animation = clipAnimation(project, clip);
  const seconds = animation ? animationDuration(animation) : 0;
  return seconds > 0 ? Math.max(1, Math.ceil(seconds * compositionFps(comp) - 1e-6)) : 0;
}

/** The frame after the last item or clip ends: how long one run of the composition is. */
export function compositionLength(project: Project, comp: CompositionAsset): number {
  let end = 0;
  for (const item of comp.items) if (itemStands(comp, item)) end = Math.max(end, itemEnd(item));
  for (const clip of comp.clips) end = Math.max(end, clip.start + clipLength(project, comp, clip));
  return end;
}

/**
 * The video the item's own object wears through its texture – the override's, else its sprite's. A
 * video worn further down a container item runs with the item but gives it no length.
 */
export function itemVideo(project: Project, comp: CompositionAsset, item: CompositionItem): VideoRef | null {
  const object = itemObject(comp, item);
  const asset = object && findAsset(project, object.assetId);
  if (!object || asset?.type !== 'sprite') return null;
  return videoWorn(resolveSceneObjectProps(project, asset, object), project);
}

function videoWorn(props: PropertyBag, project: Project): VideoRef | null {
  const texture = props.texture;
  return typeof texture === 'string' ? findVideo(project, texture) : null;
}

/**
 * The length an item with `fitMedia` takes: from its start to the end of its video, or of what an
 * audio item plays; the default while it has none or its length is not known yet.
 */
export function fittedLength(project: Project, comp: CompositionAsset, item: CompositionItem): number {
  const fps = compositionFps(comp);
  const video = item.audio ? null : itemVideo(project, comp, item);
  const seconds = item.audio
    ? audioSeconds(project, item.audio) - item.offset
    : video
      ? videoLengthOf(video) - item.offset
      : 0;
  return seconds > 0 ? Math.max(1, Math.round(seconds * fps)) : Math.round(DEFAULT_ITEM_SECONDS * fps);
}

/**
 * A composition placed in a running game that `Play composition` started; kept on its entity, out of
 * `props`. Its items exist only while it plays.
 */
export interface CompositionPlayer {
  /** Tells a start apart from the one before, so a restart builds every item anew. */
  run: number;
  /** Seconds since its first frame. */
  time: number;
  /** Whether the first frame was shown: the clock runs from the tick after the start. */
  shown: boolean;
  /** Set by `Stop composition`: the next tick takes its items away. */
  stopped: boolean;
  /** The behaviors that played it; they hear `On composition finished` wherever it stands. */
  startedBy: object[];
}

let runCount = 0;

/**
 * Starts the composition placed as `holder` from its first frame. One already playing plays on, unless
 * `restart` – or a `Stop composition` this tick – asks to start over.
 */
export function playComposition(
  holder: { composition?: CompositionPlayer },
  restart = false,
  startedBy?: object,
): void {
  const playing = holder.composition;
  if (playing && !playing.stopped && !restart) {
    if (startedBy && !playing.startedBy.includes(startedBy)) playing.startedBy.push(startedBy);
    return;
  }
  holder.composition = {
    run: ++runCount,
    time: 0,
    shown: false,
    stopped: false,
    startedBy: startedBy ? [startedBy] : [],
  };
}

export function stopComposition(holder: { composition?: CompositionPlayer }): void {
  if (holder.composition) holder.composition.stopped = true;
}

/** The frame a player's time stands on; the small guard keeps a time of exactly `n / fps` on frame n. */
export function compositionFrameAt(comp: CompositionAsset, time: number): number {
  return Math.floor(time * compositionFps(comp) + 1e-6);
}

/** A clip playing on an item, at `frame` of its animation – held on the last one once it is through. */
export interface ClipAt {
  clip: CompositionClip;
  animation: AnimationAsset;
  frame: number;
}

/** An item alive at a frame of the composition. */
export interface LiveItem {
  item: CompositionItem;
  object: SceneObject;
  /** The index of its track, 0 the first. */
  track: number;
  /** Seconds since the item began. */
  time: number;
  /** Seconds into what its videos play: `time` past the item's offset. */
  videoTime: number;
  /** How loud its videos are heard, 0 to 1: its fades at this frame. */
  level: number;
  /** In the order they started; a later one writes over an earlier one. */
  clips: ClipAt[];
}

/**
 * Everything alive at `frame`, in drawing order: the last track first, so the first lies in front –
 * the order the items take below the composition, which decides between equal `z`. An item lives
 * from its start up to, not including, its end; one on a track that is gone, or whose object is gone,
 * does not. Items of a track that overlap both live, the later one in front. A clip plays on every
 * item of its track it overlaps once it has started, and holds its last frame for as long as that
 * item lasts.
 */
export function compositionAt(project: Project, comp: CompositionAsset, frame: number): LiveItem[] {
  const fps = compositionFps(comp);
  const tracks = new Map(comp.tracks.map((t, index) => [t.id, index]));
  const clips = comp.clips
    .map((clip, order) => ({ clip, order, animation: clipAnimation(project, clip) }))
    .filter((c) => c.animation !== undefined && c.clip.start <= frame)
    .sort((a, b) => a.clip.start - b.clip.start || a.order - b.order);
  const live: { item: LiveItem; order: number }[] = [];
  comp.items.forEach((item, order) => {
    const track = tracks.get(item.track);
    const object = itemObject(comp, item);
    if (track === undefined || !object || frame < item.start || frame >= itemEnd(item)) return;
    const time = (frame - item.start) / fps;
    const playing: ClipAt[] = [];
    for (const { clip, animation } of clips) {
      const end = clip.start + clipLength(project, comp, clip);
      if (clip.track !== item.track || clip.start >= itemEnd(item) || end <= item.start) continue;
      const at = ((frame - clip.start) / fps) * fpsOf(animation!);
      playing.push({ clip, animation: animation!, frame: Math.min(at, frameCount(animation!) - 1) });
    }
    const level = fadeLevel(item, frame - item.start);
    live.push({
      item: { item, object, track, time, videoTime: item.offset + time, level, clips: playing },
      order,
    });
  });
  live.sort(
    (a, b) => b.item.track - a.item.track || a.item.item.start - b.item.item.start || a.order - b.order,
  );
  return live.map((l) => l.item);
}

/**
 * The composition at `frame` as objects a view draws: a copy holding only what is alive, in drawing
 * order, each with its clips' rows and its video time written over it. Every sprite of an item that
 * wears a video and is not driven by hand is driven at the item's video time. Nothing here touches
 * the document.
 */
export function compositionHolder(project: Project, comp: CompositionAsset, frame: number): CompositionAsset {
  return holderOf(project, comp, compositionAt(project, comp, frame));
}

/** How far ahead of where a composition stands the items about to begin are written out, in seconds. */
const AHEAD_SECONDS = 1;

/**
 * The items that begin within `seconds` after `frame` – once it loops, those at the start too when
 * the end is that close – each written out as at its first frame, like `compositionHolder` does. A
 * view opens their videos ahead of the cut: opened when its item began, a video had no picture yet,
 * and the cut showed nothing until it loaded. One alive at `frame` is left out, it wears its own time.
 */
export function compositionAhead(
  project: Project,
  comp: CompositionAsset,
  frame: number,
  seconds = AHEAD_SECONDS,
): CompositionAsset {
  const until = frame + Math.ceil(seconds * compositionFps(comp));
  const length = compositionLength(project, comp);
  const wrapped = compositionLoops(comp) && until >= length ? until - length : -1;
  const coming = comp.items.filter(
    (item) => (item.start > frame && item.start <= until) || item.start <= wrapped,
  );
  if (coming.length === 0) return holderOf(project, comp, []);
  const alive = new Set(compositionAt(project, comp, frame).map((l) => l.item.id));
  const live = coming
    .filter((item) => !alive.has(item.id))
    .flatMap((item) => compositionAt(project, comp, item.start).filter((l) => l.item.id === item.id));
  return holderOf(project, comp, live);
}

function holderOf(project: Project, comp: CompositionAsset, live: LiveItem[]): CompositionAsset {
  const holder: CompositionAsset = {
    ...comp,
    props: { ...comp.props },
    objects: live.map((l) => copyOf(l.object)),
  };
  const curves = curveLookup(project);
  for (const item of live) {
    driveVideos(project, holder, item);
    for (const clip of item.clips) {
      const at: ReadAt = { frame: clip.frame, reverse: false, curves };
      for (const track of clip.animation.tracks) {
        const id = rowTarget(project, holder, item.object.id, clip.animation, track);
        if (id) writeObjectRow(project, holder, id, track, at);
      }
    }
  }
  return holder;
}

/**
 * What a composition placed as `placementId` shows in the editor: its first frame, as objects below
 * the placement with the ids its items take in a run. Stored nowhere – a placement is sealed, and
 * its items are edited in the composition only.
 */
export function placedPicture(project: Project, comp: CompositionAsset, placementId: string): SceneObject[] {
  return compositionHolder(project, comp, 0).objects.flatMap((o) =>
    placedWithContent(project, o, contentId(placementId, o.id), placementId),
  );
}

const record = (value: unknown): Record<string, unknown> =>
  typeof value === 'object' && value !== null ? (value as Record<string, unknown>) : {};
const text = (value: unknown) => (typeof value === 'string' ? value : '');
const finite = (value: unknown) => (typeof value === 'number' && Number.isFinite(value) ? value : NaN);
const list = (raw: unknown): Record<string, unknown>[] => (Array.isArray(raw) ? raw.map(record) : []);

/** Held to what a track may be: ids no other has, a name, and at least one track. */
export function heldCompositionTracks(raw: unknown): CompositionTrack[] {
  const tracks: CompositionTrack[] = [];
  for (const t of list(raw)) {
    const id = text(t.id);
    if (!id || tracks.some((other) => other.id === id)) continue;
    tracks.push({ id, name: text(t.name) || `Track ${tracks.length + 1}` });
  }
  return tracks.length > 0 ? tracks : [newCompositionTrack(1)];
}

/**
 * Held to what an item may be: the id of an object it spawns and no other item does – an audio item
 * an id of its own and what it plays –, a track of the composition, a start in whole frames from 0,
 * at least a frame long, a finite offset, fades in whole frames above 0 and curves there are.
 */
export function heldCompositionItems(
  raw: unknown,
  tracks: readonly CompositionTrack[],
  objects: readonly SceneObject[],
): CompositionItem[] {
  const known = new Set(tracks.map((t) => t.id));
  const placed = new Set(objects.map((o) => o.id));
  const items: CompositionItem[] = [];
  for (const i of list(raw)) {
    const id = text(i.id);
    const start = Math.round(finite(i.start));
    const length = Math.round(finite(i.length));
    const audio = heldAudio(i.audio);
    if (!id || (audio ? placed.has(id) : !placed.has(id))) continue;
    if (items.some((other) => other.id === id) || !known.has(text(i.track))) continue;
    if (!(start >= 0) || !(length >= 1)) continue;
    const offset = finite(i.offset);
    const item: CompositionItem = {
      id,
      track: text(i.track),
      start,
      length,
      offset: Number.isFinite(offset) ? offset : 0,
    };
    if (i.fitMedia === true) item.fitMedia = true;
    if (audio) item.audio = audio;
    const fadeIn = Math.round(finite(i.fadeIn));
    const fadeOut = Math.round(finite(i.fadeOut));
    if (fadeIn > 0) item.fadeIn = fadeIn;
    if (fadeOut > 0) item.fadeOut = fadeOut;
    if (isFadeCurve(i.curveIn)) item.curveIn = i.curveIn;
    if (isFadeCurve(i.curveOut)) item.curveOut = i.curveOut;
    items.push(item);
  }
  return items;
}

const AUDIO_KINDS: readonly CompositionAudioKind[] = ['sound', 'pattern', 'song'];

/** Held to what an audio item plays: a kind, an asset id kept even once the asset is gone, a volume. */
function heldAudio(raw: unknown): CompositionAudio | null {
  if (typeof raw !== 'object' || raw === null) return null;
  const a = raw as Record<string, unknown>;
  const kind = AUDIO_KINDS.find((k) => k === a.kind);
  if (!kind || !text(a.asset)) return null;
  const volume = finite(a.volume);
  return {
    kind,
    asset: text(a.asset),
    volume: Number.isFinite(volume) ? Math.max(ITEM_MIN_DB, Math.min(SOUND_ITEM_MAX_DB, volume)) : 0,
    bus: text(a.bus),
  };
}

/** Held like an item; an animation that is gone is kept, as an animation keeps a root that is gone. */
export function heldCompositionClips(raw: unknown, tracks: readonly CompositionTrack[]): CompositionClip[] {
  const known = new Set(tracks.map((t) => t.id));
  const clips: CompositionClip[] = [];
  for (const c of list(raw)) {
    const id = text(c.id);
    const start = Math.round(finite(c.start));
    if (!id || clips.some((other) => other.id === id) || !known.has(text(c.track)) || !(start >= 0)) continue;
    clips.push({ id, track: text(c.track), animation: text(c.animation), start });
  }
  return clips;
}

function copyOf(object: SceneObject): SceneObject {
  return {
    ...object,
    position: { ...object.position },
    overrides: { ...object.overrides },
    instanceEdits: structuredClone(object.instanceEdits),
  };
}

function driveVideos(project: Project, holder: CompositionAsset, live: LiveItem): void {
  const below = resolveObjects(project, holder).filter(
    (o) => splitInstanceId(o.id)?.instanceId === live.object.id,
  );
  for (const object of [holder.objects.find((o) => o.id === live.object.id)!, ...below]) {
    const asset = findAsset(project, object.assetId);
    if (asset?.type !== 'sprite') continue;
    const props = resolveSceneObjectProps(project, asset, object);
    const own = object.id === live.object.id;
    if (!videoWorn(props, project) || (!own && videoDrivenOf(props))) continue;
    const ref = objectRef(project, holder, object.id);
    ref?.setOverride(VIDEO_CLOCK_KEY, 'driven');
    ref?.setOverride(VIDEO_TIME_KEY, live.videoTime);
    if (live.level < 1) ref?.setOverride(VIDEO_LEVEL_KEY, videoLevelOf(props) * live.level);
  }
}

/**
 * The object of the holder a row of a clip aims at on the item `rootId`: the item itself, the object
 * at the row's template path below it, or else the one its names lead to – the way `Play animation`
 * finds a target on an object built from another container.
 */
function rowTarget(
  project: Project,
  holder: CompositionAsset,
  rootId: string,
  animation: AnimationAsset,
  track: AnimationTrack,
): string | undefined {
  if (track.target === '') return rootId;
  const objects = resolveObjects(project, holder);
  const byPath = instanceChildId(rootId, track.target);
  if (objects.some((o) => o.id === byPath)) return byPath;
  const names = targetNamePath(project, findAsset(project, animation.root), track.target);
  let at: string | undefined = rootId;
  for (const name of names ?? []) {
    const found = objects.filter(
      (o) => o.parentId === at && objectName(o, findAsset(project, o.assetId)) === name,
    );
    at = found.length === 1 ? found[0]!.id : undefined;
    if (!at) return undefined;
  }
  return names ? at : undefined;
}