feuerwerk/src/viewport/VideoLibrary.ts

/**
 * The players behind every video a sprite wears (#402): one `<video>` per key (see `videoKey`), its
 * picture a Pixi texture that Chrome refreshes with every decoded frame. Two sprites wearing the
 * same clip share a player and show the same picture.
 *
 * In the editor a video plays without sound, as a preview. A run starts every player over at its
 * clip's start and lets it be heard along the master routing of its video; a pause holds them, and
 * the end of the run makes them quiet previews again.
 *
 * A sprite whose Video clock is driven (#404) has a player of its own instead, which shows the
 * picture at its Video time and leaves runs and pauses to it (see `wear` and `videoFollow.ts`).
 */
import { Assets, Texture, VideoSource } from 'pixi.js';
import type { PropertyBag } from '../core/model/Properties';
import type { TextureRef } from '../core/model/texture';
import {
  drivenTimeOf,
  drivenVideoKey,
  isDrivenVideoKey,
  isVideoKey,
  parseVideoKey,
  type VideoPlay,
} from '../core/model/video';
import { videoDrivenOf, videoTimeOf } from '../core/properties/videos';
import { FileLibrary } from './FileLibrary';
import { soundLibrary } from './SoundLibrary';
import { followVideo, readSpeed, type SpeedReading } from './videoFollow';
import { chaseSeeks, seekVideo, type SeekChase } from './videoSeek';

const VIDEO_BASE = '/game-maker/data/videos';

/** Where the MP4 of a video asset's `file` is served. */
export function videoFileUrl(file: string): string {
  return `${VIDEO_BASE}/${file}`;
}
/** How close to its end a clip counts as there: half a frame at 30 fps. */
const END_SLACK = 1 / 60;
/** A player nothing has asked for in this long is let go: no sprite wears it any more. */
const IDLE_MS = 2000;
/**
 * A driven player (#404) nothing has asked for in this long is held and quiet: its wearer is gone,
 * like the sprite of a composition item that has ended, or no longer drawn. It runs on by itself
 * otherwise, heard, until it is let go.
 */
const HUSH_MS = 100;

/**
 * When the frame being drawn began, in milliseconds like `performance.now()`. The ticker moves every
 * clock on by the step between these, so a driven video's time does too; `performance.now()` read
 * while drawing lags it by whatever the frame did before. A busy page lags more in some frames
 * than in others, and 45 ms more at one end of a window of 500 ms reads a 1x time as 1.09.
 */
function frameTime(): number {
  const now =
    typeof document !== 'undefined' && 'timeline' in document ? document.timeline.currentTime : null;
  return typeof now === 'number' ? now : performance.now();
}

/**
 * Whether a wearer lets its video be heard: in a game along the master routing of the video, in a
 * preview of a tab straight into the master, as every tab's preview is (#368).
 */
export type VideoHearing = 'none' | 'preview' | 'game';

/** What the wearer of a driven player (#404) asks of it, see `wear`; its seeks, see `videoSeek.ts`. */
interface Drive extends SeekChase {
  /** Seconds into the clip. */
  time: number;
  /** Whether and how the wearer lets it be heard, when it plays at about 1x. */
  heard: VideoHearing;
  reading: SpeedReading | null;
}

interface Player {
  play: VideoPlay;
  element: HTMLVideoElement;
  /** Made once the video has shown its first frame, see `openVideo`. */
  texture: Texture | null;
  /** The sound on its way into its bus, made the first time a run lets it be heard. */
  gain: GainNode | null;
  gone: boolean;
  /** When a sprite last asked for its picture, by `performance.now()`. */
  asked: number;
  /** Set when its wearer sets the time rather than the video itself. */
  drive: Drive | null;
}

export class VideoLibrary {
  private readonly players = new Map<string, Player>();
  private readonly listeners = new Set<() => void>();
  private heard = false;
  private paused = false;
  private sweeper: ReturnType<typeof setInterval> | null = null;
  private husher: ReturnType<typeof setInterval> | null = null;
  /** When the page was last shown, by `performance.now()`; see `sweep`. */
  private shown = 0;

  constructor() {
    if (typeof document === 'undefined') return;
    document.addEventListener('visibilitychange', () => {
      if (!document.hidden) this.shown = performance.now();
    });
  }

  /** Runs `fn` whenever a player has its first picture; returns the unsubscribe. */
  onChange(fn: () => void): () => void {
    this.listeners.add(fn);
    return () => this.listeners.delete(fn);
  }

  /** The picture of a playing video, made on the first ask; null until it has one. */
  get(key: string): Texture | null {
    const player = this.players.get(key) ?? this.start(key);
    if (!player) return null;
    player.asked = performance.now();
    return player.texture;
  }

  /**
   * What a sprite wears (#404): with its clock driven, a player of its own that shows the picture
   * at its Video time; otherwise the ref as it is. `wearer` names the sprite where it is drawn, so
   * the same object in two views drives two players. Asked every frame, like `get`, and it counts
   * as an ask: a sprite switched off wears its player without drawing it, and one let go for that
   * loaded again when it was switched back on, white until then.
   */
  wear(ref: TextureRef, props: PropertyBag, wearer: string, heard: VideoHearing): TextureRef {
    if (!videoDrivenOf(props) || !isVideoKey(ref.file)) return ref;
    const key = drivenVideoKey(ref.file, wearer);
    const time = videoTimeOf(props);
    const player = this.players.get(key) ?? this.start(key, time);
    if (!player?.drive) return ref;
    player.asked = performance.now();
    const drive = player.drive;
    drive.time = time;
    drive.heard = heard;
    drive.reading = readSpeed(drive.reading, time, frameTime());
    if (player.texture) this.follow(player, drive);
    return { ...ref, file: key };
  }

  /** Whether the player behind `key` has its first picture; it does not count as an ask. */
  hasPicture(key: string): boolean {
    return this.players.get(key)?.texture != null;
  }

  /**
   * Whether the player behind `key` shows the picture its wearer asked for last: it has one and no
   * seek is under way. Until a seek has landed, its texture still holds the picture it left.
   */
  showsAsked(key: string): boolean {
    const player = this.players.get(key);
    return player?.texture != null && !player.drive?.landing;
  }

  /**
   * Whether a driven player (#404) the view `view` wears (see `wear`) is still without its first
   * picture. A run waits for them at its start: the Game tab opens players of its own at Play.
   */
  loading(view: string): boolean {
    const mark = `@${view}:`;
    for (const [key, player] of this.players) if (!player.texture && key.includes(mark)) return true;
    return false;
  }

  /** A run begins: every player from its start, heard. A driven one goes where its wearer says. */
  startRun(): void {
    this.heard = true;
    this.paused = false;
    for (const player of this.freePlayers()) {
      this.hear(player);
      this.rewind(player);
    }
  }

  /** The run is over: back to quiet previews, from their start. */
  endRun(): void {
    this.heard = false;
    this.paused = false;
    for (const player of this.freePlayers()) {
      player.element.muted = true;
      this.rewind(player);
    }
  }

  setPaused(paused: boolean): void {
    this.paused = paused;
    for (const player of this.freePlayers())
      if (paused) player.element.pause();
      else void player.element.play().catch(() => {});
  }

  private freePlayers(): Player[] {
    return [...this.players.values()].filter((player) => !player.drive);
  }

  private follow(player: Player, drive: Drive): void {
    const { element, play } = player;
    const step = followVideo({
      target: drivenTimeOf(play, drive.time, element.duration),
      speed: drive.reading?.speed ?? 0,
      current: element.currentTime,
      // A video that ran to the end of its file starts over from its first picture on play().
      end: element.ended ? element.currentTime : Math.min(play.end, element.duration),
    });
    if (step.seek !== null) seekVideo(element, drive, step.seek);
    if (step.kind === 'hold') {
      element.muted = true;
      if (!element.paused) element.pause();
      return;
    }
    if (Math.abs(element.playbackRate - step.rate) > 0.001) element.playbackRate = step.rate;
    if (step.heard && drive.heard !== 'none') this.hear(player, drive.heard === 'game');
    else element.muted = true;
    if (element.paused) void element.play().catch(() => {});
  }

  /** Lets every player go: a session that ends takes its videos along. */
  clear(): void {
    for (const player of this.players.values()) this.drop(player);
    this.players.clear();
    this.stopSweeping();
  }

  /**
   * Drops the players nobody asked for lately; a paused run asks for nothing and keeps them. So
   * does a hidden page: Chrome draws no frames for it while its timers go on, and a sprite that
   * still wears a video would lose it. The time hidden does not count as idle.
   */
  private sweep(): void {
    if (this.paused || document.hidden) return;
    const now = performance.now();
    for (const [key, player] of this.players)
      if (now - Math.max(player.asked, this.shown) > IDLE_MS) {
        this.drop(player);
        this.players.delete(key);
      }
    if (this.players.size === 0) this.stopSweeping();
  }

  /**
   * Holds and quiets the driven players nobody asked for lately, see `HUSH_MS`. One still without
   * its first picture plays on, without sound: it only gets one by playing (see `openVideo`).
   */
  private hush(): void {
    const now = performance.now();
    for (const player of this.players.values()) {
      if (!player.drive || !player.texture || now - player.asked <= HUSH_MS) continue;
      player.element.muted = true;
      if (!player.element.paused) player.element.pause();
    }
  }

  private stopSweeping(): void {
    if (this.sweeper !== null) clearInterval(this.sweeper);
    if (this.husher !== null) clearInterval(this.husher);
    this.sweeper = null;
    this.husher = null;
  }

  /** What each player is doing (test hook). */
  debugInfo(): {
    key: string;
    time: number;
    paused: boolean;
    muted: boolean;
    ready: boolean;
    rate: number;
    seeking: boolean;
    /** How fast its wearer's time runs, as read for a driven player; 0 for a free one. */
    speed: number;
    /** Whether a driven player still waits for the picture of a seek, see `seek`. */
    landing: boolean;
  }[] {
    return [...this.players].map(([key, { element, texture, drive }]) => ({
      key,
      time: element.currentTime,
      paused: element.paused,
      muted: element.muted,
      ready: texture !== null,
      rate: element.playbackRate,
      seeking: element.seeking,
      speed: drive?.reading?.speed ?? 0,
      landing: drive?.landing ?? false,
    }));
  }

  /** A driven player (#404) opens at the picture its wearer asks for, `time` seconds into the clip. */
  private start(key: string, time = 0): Player | null {
    const play = parseVideoKey(key);
    if (!play) return null;
    const drive: Drive | null = isDrivenVideoKey(key)
      ? { time, heard: 'none', reading: null, wanted: null, landing: false }
      : null;
    const at = drive ? drivenTimeOf(play, time, NaN) : play.start;
    const player: Player = {
      play,
      element: openVideo(videoFileUrl(play.file), at, (texture) => {
        if (player.gone) return texture.destroy(true);
        player.texture = texture;
        if (drive) this.follow(player, drive);
        // Started even in a paused run, and held once there is a picture.
        else if (this.paused) player.element.pause();
        this.notify();
      }),
      texture: null,
      gain: null,
      gone: false,
      asked: performance.now(),
      drive,
    };
    this.players.set(key, player);
    this.sweeper ??= setInterval(() => this.sweep(), IDLE_MS);
    if (drive) this.husher ??= setInterval(() => this.hush(), HUSH_MS);
    if (drive)
      chaseSeeks(
        player.element,
        drive,
        () => player.gone,
        () => player.texture?.source.update(),
      );
    else {
      this.watch(player);
      if (this.heard) this.hear(player);
    }
    return player;
  }

  /**
   * Sends a clip back to its start at its end, or holds it there when it does not repeat. A clip
   * that runs to the end of the video is sent back when the video ends; the whole video loops by
   * itself.
   */
  private watch(player: Player): void {
    const { element, play } = player;
    element.loop = play.loop && play.start === 0 && !Number.isFinite(play.end);
    element.addEventListener('ended', () => {
      if (player.gone || !play.loop) return;
      element.currentTime = play.start;
      void element.play().catch(() => {});
    });
    if (!Number.isFinite(play.end)) return;
    const check = (_now: number, frame: VideoFrameCallbackMetadata) => {
      if (player.gone) return;
      if (frame.mediaTime >= play.end - END_SLACK || element.currentTime >= play.end) {
        if (play.loop) element.currentTime = play.start;
        else element.pause();
      }
      element.requestVideoFrameCallback(check);
    };
    element.requestVideoFrameCallback(check);
  }

  /** Along the master routing of its video where `routed`, else straight into the master. */
  private hear(player: Player, routed = true): void {
    player.gain ??= soundLibrary.mediaInput(player.element, player.play.file, routed);
    if (player.gain) player.gain.gain.value = player.play.volume;
    player.element.muted = false;
  }

  private rewind(player: Player): void {
    player.element.currentTime = player.play.start;
    if (!this.paused) void player.element.play().catch(() => {});
  }

  private drop(player: Player): void {
    player.gone = true;
    player.element.pause();
    if (player.gain) soundLibrary.releaseMedia(player.gain);
    player.element.removeAttribute('src');
    player.element.load();
    // The texture itself stays whole: a wearer that has not synced since still draws it, and a
    // destroyed one has no source and threw in the batcher. It shows nothing until the wearer asks
    // again. The video source is only unloaded, not destroyed: a shader may still hold it until
    // its wearer syncs, and Pixi warns about every destroyed source still bound to one.
    const texture = player.texture;
    if (!texture) return;
    const video = texture.source as VideoSource;
    texture.source = Texture.EMPTY.source;
    video.autoUpdate = false;
    video.unload();
  }

  private notify(): void {
    for (const listener of this.listeners)
      try {
        listener();
      } catch (error) {
        console.error('[videos] a listener failed', error);
      }
  }
}

export const videoLibrary = new VideoLibrary();

/**
 * A `<video>` of `url`, started without sound at `at` seconds, and its picture as a texture once it
 * has shown its first frame. Chrome has no picture of a video that never played, and a Pixi source
 * drawn before it has one stays empty for good (measured with Pixi 8.20: playing and `update()`
 * afterwards changed nothing, a source made after the first frame showed every frame). The caller
 * holds the element where it wants it once `onPicture` has run.
 */
export function openVideo(url: string, at: number, onPicture: (texture: Texture) => void): HTMLVideoElement {
  const element = document.createElement('video');
  element.muted = true;
  element.playsInline = true;
  element.preload = 'auto';
  element.src = url;
  if (at > 0) element.currentTime = at;
  element.addEventListener(
    'loadeddata',
    () => {
      element.requestVideoFrameCallback(() =>
        onPicture(new Texture({ source: new VideoSource({ resource: element, autoPlay: false }) })),
      );
      void element.play().catch(() => {});
    },
    { once: true },
  );
  return element;
}

/** The stills the worker cut from each video, for the picker (see `posterFileOf`). */
class PosterLibrary extends FileLibrary<Texture> {
  private readonly loaded = new Map<string, Texture>();

  constructor() {
    super(VIDEO_BASE, 'posters');
  }

  /** The still if it is in memory, otherwise null while it loads. */
  load(file: string): Texture | null {
    const texture = this.loaded.get(file);
    if (!texture) this.startLoading(file);
    return texture ?? null;
  }

  protected fetchFile(url: string): Promise<Texture> {
    return Assets.load<Texture>(url);
  }

  protected keep(file: string, texture: Texture): void {
    this.loaded.set(file, texture);
  }

  protected holds(file: string): boolean {
    return this.loaded.has(file);
  }
}

export const posterLibrary = new PosterLibrary();