feuerwerk/src/viewport/VideoLibrary.ts

/**
 * The players behind every video a sprite wears (#402): one `<video>` per key (see `videoKey`), its
 * picture a Pixi texture that Chrome refreshes with every decoded frame. Two sprites wearing the
 * same clip share a player and show the same picture.
 *
 * In the editor a video plays without sound, as a preview. A run starts every player over at its
 * clip's start and lets it be heard along the master routing of its video; a pause holds them, and
 * the end of the run makes them quiet previews again.
 *
 * A sprite whose Video clock is driven (#404) has a player of its own instead, which shows the
 * picture at its Video time and leaves runs and pauses to it (see `wear` and `videoFollow.ts`).
 */
import { Assets, Texture, VideoSource } from 'pixi.js';
import type { PropertyBag } from '../core/model/Properties';
import { dataPath } from '../core/site';
import type { TextureRef } from '../core/model/texture';
import {
  drivenTimeOf,
  drivenVideoKey,
  isDrivenVideoKey,
  isVideoKey,
  parseVideoKey,
  type VideoPlay,
} from '../core/model/video';
import { videoDrivenOf, videoLevelOf, videoTimeOf } from '../core/properties/videos';
import { FileLibrary } from './FileLibrary';
import { soundLibrary } from './SoundLibrary';
import type { HeardVideo } from './SteadySound';
import {
  followVideo,
  PICTURE_NUDGE,
  pictureShown,
  readSpeed,
  type ShownPicture,
  showsPictureOf,
  type SpeedReading,
} from './videoFollow';
import { chaseSeeks, seekVideo, type SeekChase } from './videoSeek';

const VIDEO_BASE = dataPath('videos');

/** Where the MP4 of a video asset's `file` is served. */
export function videoFileUrl(file: string): string {
  return `${VIDEO_BASE}/${file}`;
}
/** How close to its end a clip counts as there: half a frame at 30 fps. */
const END_SLACK = 1 / 60;
/** A player nothing has asked for in this long is let go: no sprite wears it any more. */
const IDLE_MS = 2000;
/**
 * A player nothing has asked for in this long is held and quiet: its wearer is gone, like a sprite
 * destroyed or the sprite of a composition item that has ended, or no longer drawn. It runs on by
 * itself otherwise, heard, until it is let go. A free one (#420) plays on from there once asked again.
 */
const HUSH_MS = 100;
/** Seconds a driven video's level takes to get most of the way to a new one, see `hear`. */
const LEVEL_GLIDE = 0.015;

/**
 * When the frame being drawn began, in milliseconds like `performance.now()`. The ticker moves every
 * clock on by the step between these, so a driven video's time does too; `performance.now()` read
 * while drawing lags it by whatever the frame did before. A busy page lags more in some frames
 * than in others, and 45 ms more at one end of a window of 500 ms reads a 1x time as 1.09.
 */
function frameTime(): number {
  const now =
    typeof document !== 'undefined' && 'timeline' in document ? document.timeline.currentTime : null;
  return typeof now === 'number' ? now : performance.now();
}

/**
 * Whether a wearer lets its video be heard: in a game along the master routing of the video, in a
 * preview of a tab straight into the master, as every tab's preview is (#368).
 */
export type VideoHearing = 'none' | 'preview' | 'game';

/** What the wearer of a driven player (#404) asks of it, see `wear`; its seeks, see `videoSeek.ts`. */
interface Drive extends SeekChase {
  /** Seconds into the clip. */
  time: number;
  /** Whether and how the wearer lets it be heard, when it plays at about 1x. */
  heard: VideoHearing;
  /** How loud the wearer hears it, 0–1, on top of the video's own volume: its Video volume. */
  level: number;
  reading: SpeedReading | null;
  /** The picture it was held at last, as Chrome handed it over; null while it plays. */
  shown: ShownPicture | null;
}

interface Player {
  play: VideoPlay;
  element: HTMLVideoElement;
  /** Made once the video has shown its first frame, see `openVideo`. */
  texture: Texture | null;
  /** The sound on its way into its bus, made the first time a run lets it be heard. */
  gain: GainNode | null;
  /** The level its gain was last sent to. */
  level: number;
  gone: boolean;
  /** When a sprite last asked for its picture, by `performance.now()`. */
  asked: number;
  /** Set when its wearer sets the time rather than the video itself. */
  drive: Drive | null;
  /** Whether a free player is held and quiet because nothing wears it, see `hush`. */
  hushed: boolean;
}

export class VideoLibrary {
  private readonly players = new Map<string, Player>();
  private readonly listeners = new Set<() => void>();
  private heard = false;
  private paused = false;
  /** Driven players hold at every picture asked rather than play: a recording takes each (#423). */
  private stepped = false;
  private sweeper: ReturnType<typeof setInterval> | null = null;
  private husher: ReturnType<typeof setInterval> | null = null;
  /** When the page was last shown, by `performance.now()`; see `sweep`. */
  private shown = 0;

  constructor() {
    if (typeof document === 'undefined') return;
    document.addEventListener('visibilitychange', () => {
      if (!document.hidden) this.shown = performance.now();
    });
  }

  /** Runs `fn` whenever a player has its first picture; returns the unsubscribe. */
  onChange(fn: () => void): () => void {
    this.listeners.add(fn);
    return () => this.listeners.delete(fn);
  }

  /** The picture of a playing video, made on the first ask; null until it has one. */
  get(key: string): Texture | null {
    const player = this.players.get(key) ?? this.start(key);
    if (!player) return null;
    player.asked = performance.now();
    if (player.hushed) this.unhush(player);
    return player.texture;
  }

  /**
   * What a sprite wears (#404): with its clock driven, a player of its own that shows the picture
   * at its Video time; otherwise the ref as it is. `wearer` names the sprite where it is drawn, so
   * the same object in two views drives two players. Asked every frame, like `get`, and it counts
   * as an ask: a sprite switched off wears its player without drawing it, and one let go for that
   * loaded again when it was switched back on, white until then.
   */
  wear(ref: TextureRef, props: PropertyBag, wearer: string, heard: VideoHearing): TextureRef {
    if (!videoDrivenOf(props) || !isVideoKey(ref.file)) return ref;
    const key = drivenVideoKey(ref.file, wearer);
    const time = videoTimeOf(props);
    const player = this.players.get(key) ?? this.start(key, time);
    if (!player?.drive) return ref;
    player.asked = performance.now();
    const drive = player.drive;
    drive.time = time;
    drive.heard = heard;
    drive.level = videoLevelOf(props);
    drive.reading = readSpeed(drive.reading, time, frameTime());
    if (player.texture) this.follow(player, drive);
    return { ...ref, file: key };
  }

  /** Whether the player behind `key` has its first picture; it does not count as an ask. */
  hasPicture(key: string): boolean {
    return this.players.get(key)?.texture != null;
  }

  /**
   * Whether the player behind `key` shows the picture its wearer asked for last: it has one and no
   * seek is under way. Until a seek has landed, its texture still holds the picture it left.
   */
  showsAsked(key: string): boolean {
    const player = this.players.get(key);
    return player?.texture != null && !player.drive?.landing;
  }

  /**
   * Whether a driven player (#404) the view `view` wears (see `wear`) is still without its first
   * picture. A run waits for them at its start: the Game tab opens players of its own at Play.
   */
  loading(view: string): boolean {
    const mark = `@${view}:`;
    for (const [key, player] of this.players) if (!player.texture && key.includes(mark)) return true;
    return false;
  }

  /**
   * A run begins: every player from its start, heard; a hushed one once it is worn again. A driven
   * one goes where its wearer says.
   */
  startRun(): void {
    this.heard = true;
    this.paused = false;
    for (const player of this.freePlayers()) {
      if (!player.hushed) this.hear(player);
      this.rewind(player);
    }
  }

  /** The run is over: back to quiet previews, from their start. */
  endRun(): void {
    this.heard = false;
    this.paused = false;
    for (const player of this.freePlayers()) {
      player.element.muted = true;
      this.rewind(player);
    }
  }

  setPaused(paused: boolean): void {
    this.paused = paused;
    for (const player of this.freePlayers())
      if (paused) player.element.pause();
      else if (!player.hushed) void player.element.play().catch(() => {});
  }

  private freePlayers(): Player[] {
    return [...this.players.values()].filter((player) => !player.drive);
  }

  /**
   * While a recording not in real time goes (#423), every driven player holds at the picture its
   * wearer asks for and seeks to the next: the frames come slower than it would play, and each has
   * to show its own picture. Their sound is the recording's business then.
   */
  setStepped(stepped: boolean): void {
    this.stepped = stepped;
  }

  /** What a driven player would let be heard in a game now (#423); null for one that is not. */
  heardOf(key: string): HeardVideo | null {
    const player = this.players.get(key);
    const drive = player?.drive;
    if (!player || !drive || player.gone || drive.heard !== 'game') return null;
    const { play, element } = player;
    return {
      key,
      file: play.file,
      time: drivenTimeOf(play, drive.time, element.duration),
      level: play.volume * drive.level,
      ...(play.bus !== undefined && { bus: play.bus }),
    };
  }

  private follow(player: Player, drive: Drive): void {
    const { element, play } = player;
    const step = followVideo({
      target: drivenTimeOf(play, drive.time, element.duration),
      speed: this.stepped ? 0 : (drive.reading?.speed ?? 0),
      current: element.currentTime,
      // A video that ran to the end of its file starts over from its first picture on play().
      end: element.ended ? element.currentTime : Math.min(play.end, element.duration),
    });
    // Held frame by frame for a recording (#423), a seek within the picture shown brings no other.
    const shown =
      this.stepped &&
      step.seek !== null &&
      element.paused &&
      !drive.landing &&
      showsPictureOf(drive.shown, step.seek);
    if (step.seek !== null && !shown)
      seekVideo(element, drive, this.stepped ? step.seek + PICTURE_NUDGE : step.seek);
    if (step.kind === 'hold') {
      element.muted = true;
      if (!element.paused) element.pause();
      return;
    }
    drive.shown = null;
    if (Math.abs(element.playbackRate - step.rate) > 0.001) element.playbackRate = step.rate;
    if (step.heard && drive.heard !== 'none') this.hear(player, drive.heard === 'game');
    else element.muted = true;
    if (element.paused) void element.play().catch(() => {});
  }

  /** Lets every player go: a session that ends takes its videos along. */
  clear(): void {
    for (const player of this.players.values()) this.drop(player);
    this.players.clear();
    this.stopSweeping();
  }

  /**
   * Drops the players nobody asked for lately; a paused run asks for nothing and keeps them. So
   * does a hidden page: Chrome draws no frames for it while its timers go on, and a sprite that
   * still wears a video would lose it. The time hidden does not count as idle.
   */
  private sweep(): void {
    if (this.paused || document.hidden) return;
    const now = performance.now();
    for (const [key, player] of this.players)
      if (now - Math.max(player.asked, this.shown) > IDLE_MS) {
        this.drop(player);
        this.players.delete(key);
      }
    if (this.players.size === 0) this.stopSweeping();
  }

  /**
   * Holds and quiets the players nobody asked for lately, see `HUSH_MS`. One still without its first
   * picture plays on, without sound: it only gets one by playing (see `openVideo`). A free one is
   * left alone while the page is hidden, as by `sweep`.
   */
  private hush(): void {
    const now = performance.now();
    for (const player of this.players.values()) {
      if (player.drive) {
        if (!player.texture || now - player.asked <= HUSH_MS) continue;
      } else if (player.hushed || document.hidden || now - Math.max(player.asked, this.shown) <= HUSH_MS)
        continue;
      player.hushed = !player.drive;
      player.element.muted = true;
      if (player.texture && !player.element.paused) player.element.pause();
    }
  }

  /** A free player worn again plays on from where it was held, heard in a run. */
  private unhush(player: Player): void {
    player.hushed = false;
    if (this.heard) this.hear(player);
    if (!this.paused) void player.element.play().catch(() => {});
  }

  private stopSweeping(): void {
    if (this.sweeper !== null) clearInterval(this.sweeper);
    if (this.husher !== null) clearInterval(this.husher);
    this.sweeper = null;
    this.husher = null;
  }

  /** What each player is doing (test hook). */
  debugInfo(): {
    key: string;
    time: number;
    paused: boolean;
    muted: boolean;
    ready: boolean;
    rate: number;
    seeking: boolean;
    /** How fast its wearer's time runs, as read for a driven player; 0 for a free one. */
    speed: number;
    /** Whether a driven player still waits for the picture of a seek, see `seek`. */
    landing: boolean;
  }[] {
    return [...this.players].map(([key, { element, texture, drive }]) => ({
      key,
      time: element.currentTime,
      paused: element.paused,
      muted: element.muted,
      ready: texture !== null,
      rate: element.playbackRate,
      seeking: element.seeking,
      speed: drive?.reading?.speed ?? 0,
      landing: drive?.landing ?? false,
    }));
  }

  /** A driven player (#404) opens at the picture its wearer asks for, `time` seconds into the clip. */
  private start(key: string, time = 0): Player | null {
    const play = parseVideoKey(key);
    if (!play) return null;
    const drive: Drive | null = isDrivenVideoKey(key)
      ? { time, heard: 'none', level: 1, reading: null, wanted: null, landing: false, shown: null }
      : null;
    const at = drive ? drivenTimeOf(play, time, NaN) : play.start;
    const player: Player = {
      play,
      element: openVideo(videoFileUrl(play.file), at, (texture) => {
        if (player.gone) return texture.destroy(true);
        player.texture = texture;
        if (drive) this.follow(player, drive);
        // Started even in a paused run or when nothing wears it, and held once there is a picture.
        else if (this.paused || player.hushed) player.element.pause();
        this.notify();
      }),
      texture: null,
      gain: null,
      level: -1,
      gone: false,
      asked: performance.now(),
      drive,
      hushed: false,
    };
    this.players.set(key, player);
    this.sweeper ??= setInterval(() => this.sweep(), IDLE_MS);
    this.husher ??= setInterval(() => this.hush(), HUSH_MS);
    if (drive)
      chaseSeeks(
        player.element,
        drive,
        () => player.gone,
        (shown) => {
          player.texture?.source.update();
          if (shown !== null && player.element.paused) drive.shown = pictureShown(drive.shown, shown);
        },
      );
    else {
      this.watch(player);
      if (this.heard) this.hear(player);
    }
    return player;
  }

  /**
   * Sends a clip back to its start at its end, or holds it there when it does not repeat. A clip
   * that runs to the end of the video is sent back when the video ends; the whole video loops by
   * itself.
   */
  private watch(player: Player): void {
    const { element, play } = player;
    element.loop = play.loop && play.start === 0 && !Number.isFinite(play.end);
    element.addEventListener('ended', () => {
      if (player.gone || !play.loop) return;
      element.currentTime = play.start;
      void element.play().catch(() => {});
    });
    if (!Number.isFinite(play.end)) return;
    const check = (_now: number, frame: VideoFrameCallbackMetadata) => {
      if (player.gone) return;
      if (frame.mediaTime >= play.end - END_SLACK || element.currentTime >= play.end) {
        if (play.loop) element.currentTime = play.start;
        else element.pause();
      }
      element.requestVideoFrameCallback(check);
    };
    element.requestVideoFrameCallback(check);
  }

  /**
   * Along the master routing of its video where `routed`, else straight into the master. A driven
   * one's level glides to what its wearer asks: set once a frame, a fade stepped and clicked.
   */
  private hear(player: Player, routed = true): void {
    const made = player.gain === null;
    player.gain ??= soundLibrary.mediaInput(player.element, player.play.file, routed, player.play.bus);
    const level = player.play.volume * (player.drive?.level ?? 1);
    if (player.gain && made) player.gain.gain.value = level;
    else if (player.gain && player.level !== level)
      player.gain.gain.setTargetAtTime(level, player.gain.context.currentTime, LEVEL_GLIDE);
    player.level = level;
    player.element.muted = false;
  }

  private rewind(player: Player): void {
    player.element.currentTime = player.play.start;
    if (!this.paused && !player.hushed) void player.element.play().catch(() => {});
  }

  private drop(player: Player): void {
    player.gone = true;
    player.element.pause();
    if (player.gain) soundLibrary.releaseMedia(player.gain);
    player.element.removeAttribute('src');
    player.element.load();
    // The texture itself stays whole: a wearer that has not synced since still draws it, and a
    // destroyed one has no source and threw in the batcher. It shows nothing until the wearer asks
    // again. The video source is only unloaded, not destroyed: a shader may still hold it until
    // its wearer syncs, and Pixi warns about every destroyed source still bound to one.
    const texture = player.texture;
    if (!texture) return;
    const video = texture.source as VideoSource;
    texture.source = Texture.EMPTY.source;
    video.autoUpdate = false;
    video.unload();
  }

  private notify(): void {
    for (const listener of this.listeners)
      try {
        listener();
      } catch (error) {
        console.error('[videos] a listener failed', error);
      }
  }
}

export const videoLibrary = new VideoLibrary();

/**
 * A `<video>` of `url`, started without sound at `at` seconds, and its picture as a texture once it
 * has shown its first frame. Chrome has no picture of a video that never played, and a Pixi source
 * drawn before it has one stays empty for good (measured with Pixi 8.20: playing and `update()`
 * afterwards changed nothing, a source made after the first frame showed every frame). The caller
 * holds the element where it wants it once `onPicture` has run.
 */
export function openVideo(url: string, at: number, onPicture: (texture: Texture) => void): HTMLVideoElement {
  const element = document.createElement('video');
  element.muted = true;
  element.playsInline = true;
  element.preload = 'auto';
  element.src = url;
  if (at > 0) element.currentTime = at;
  element.addEventListener(
    'loadeddata',
    () => {
      element.requestVideoFrameCallback(() =>
        onPicture(new Texture({ source: new VideoSource({ resource: element, autoPlay: false }) })),
      );
      void element.play().catch(() => {});
    },
    { once: true },
  );
  return element;
}

/** The stills the worker cut from each video, for the picker (see `posterFileOf`). */
class PosterLibrary extends FileLibrary<Texture> {
  private readonly loaded = new Map<string, Texture>();

  constructor() {
    super(VIDEO_BASE, 'posters');
  }

  /** The still if it is in memory, otherwise null while it loads. */
  load(file: string): Texture | null {
    const texture = this.loaded.get(file);
    if (!texture) this.startLoading(file);
    return texture ?? null;
  }

  protected fetchFile(url: string): Promise<Texture> {
    return Assets.load<Texture>(url);
  }

  protected keep(file: string, texture: Texture): void {
    this.loaded.set(file, texture);
  }

  protected holds(file: string): boolean {
    return this.loaded.has(file);
  }
}

export const posterLibrary = new PosterLibrary();