feuerwerk/src/editor/video/VideoCanvas.ts

import {
  Container,
  type FederatedPointerEvent,
  type FederatedWheelEvent,
  Graphics,
  Sprite,
  type Text,
  Texture,
} from 'pixi.js';
import { AddClipCommand, SetClipPropsCommand } from '../../core/commands/clipCommands';
import type { VideoAsset, VideoClip } from '../../core/model/Asset';
import type { PropertyBag, Vec2 } from '../../core/model/Properties';
import { findAssetOf } from '../../core/model/Project';
import { clipsOf } from '../../core/model/sound';
import {
  videoLoopsOf,
  videoSpanOf,
  videoVolumeOf,
  VIDEO_END_KEY,
  VIDEO_START_KEY,
} from '../../core/properties/videos';
import { crispText, type Size, textStyle, Theme, ui, Widget, zoomFactor } from '../../ui';
import {
  clampWave,
  type ClipBand,
  clipAt,
  type CutSpan,
  draggedSpan,
  edgeAt,
  fitWave,
  rulerLabel,
  rulerStep,
  secondsToX,
  spanOf,
  storedCut,
  type WaveView,
  xToSeconds,
  zoomWave,
} from '../sound/waveView';
import { openVideo } from '../../viewport/VideoLibrary';
import type { EditorState } from '../state';
import { containRect } from '../texture/imageGrid';

const RULER_HEIGHT = 24;
/** The strip the clips stand in as bands, as in the Sound tab. */
const STRIP_HEIGHT = 20;
/** The track under it, where the picked clip's edges are dragged. */
const TRACK_HEIGHT = 32;
const TIMELINE_HEIGHT = RULER_HEIGHT + STRIP_HEIGHT + TRACK_HEIGHT;
const FIT_MARGIN = 16;
const LABEL_SPACING = 70;
/** A clip is a frame long at least; at 60 fps that is this. */
const LEAST_SPAN = 1 / 60;
const DRAG_START = 3;
/** Clip edges are stored to the millisecond. */
const STORED_RATE = 1000;
const PREVIEW_PAD = 8;
const LIT_LABEL = textStyle('small').clone();
LIT_LABEL.fill = 0x30343c;

type Gesture =
  | { kind: 'pan'; pointerX: number; panX: number }
  | { kind: 'seek' }
  | {
      kind: 'span';
      clipId: string | null;
      edge: 'start' | 'end' | null;
      from: number;
      pointerX: number;
      moved: boolean;
      span: CutSpan;
      picks?: string | null;
    };

/** The tab's own player: the video shown, with its sound, apart from the ones sprites wear. */
interface Preview {
  file: string;
  element: HTMLVideoElement;
  /** Made once the video has shown its first frame, see `openVideo`. */
  texture: Texture | null;
}

/**
 * The Video tab's canvas (#402): the video on top, a ruler in seconds under it, the clips as bands
 * in a strip, and a track where the picked clip stands between two handles. A click in the strip
 * picks a clip, a drag there or in the track spans a new one; dragging a handle moves that edge.
 * One drag is one undo step. A click in the ruler shows that moment; the wheel zooms the time, the
 * middle button pans, as in the Sound tab.
 */
/** The colour in the middle of the frame a video holds; transparent where it has none to draw. */
export function middlePixel(element: HTMLVideoElement): number[] | null {
  if (element.readyState < 2) return null;
  const context = new OffscreenCanvas(8, 8).getContext('2d', { willReadFrequently: true });
  if (!context) return null;
  context.drawImage(element, 0, 0, 8, 8);
  return [...context.getImageData(4, 4, 1, 1).data];
}

export class VideoCanvas extends Widget {
  private readonly bg = new Graphics();
  private readonly picture = new Sprite();
  private readonly overlay = new Graphics();
  private readonly labels = new Container();
  private readonly texts: Text[] = [];
  private readonly stripLabels = new Container();
  private readonly stripTexts: Text[] = [];
  private readonly hint = crispText({ text: '', style: textStyle('muted') });
  private view: WaveView = { panX: FIT_MARGIN, pxPerSecond: 100 };
  private needsFit = true;
  private assetId = '';
  private gesture: Gesture | null = null;
  private preview: Preview | null = null;
  /** The span the player is held to while it plays: the picked clip, else the whole video. */
  private playing: CutSpan | null = null;
  onTogglePlay: () => void = () => {};
  /** The player stopped by itself (the end of a clip that does not repeat). */
  onStopped: () => void = () => {};

  constructor(private readonly state: EditorState) {
    super();
    this.focusable = true;
    this.labels.eventMode = 'none';
    this.stripLabels.eventMode = 'none';
    this.hint.eventMode = 'none';
    this.picture.eventMode = 'none';
    this.addChild(this.bg, this.picture, this.overlay, this.labels, this.stripLabels, this.hint);
    this.on('pointerdown', (e: FederatedPointerEvent) => this.onPointerDown(e));
    this.on('globalpointermove', (e: FederatedPointerEvent) => this.onPointerMove(e));
    this.onPointerRelease(() => this.onPointerUp());
    this.on('wheel', (e: FederatedWheelEvent) => this.onWheel(e));
    state.on('project', () => this.invalidate());
    state.on('selection', () => {
      if (!this.isPlaying) this.seekTo(this.shownSpan()?.start ?? 0);
      this.invalidate();
    });
  }

  override measure(): Size {
    return { w: Math.max(this.minWidth, 360), h: Math.max(this.minHeight, TIMELINE_HEIGHT + 120) };
  }

  get currentAssetId(): string {
    return this.assetId;
  }

  show(assetId: string): void {
    if (this.assetId === assetId) return;
    this.stop();
    this.gesture = null;
    this.assetId = assetId;
    this.needsFit = true;
    this.redraw();
  }

  saveView(): WaveView {
    return { ...this.view };
  }

  restoreView(view: unknown): void {
    const held = view as Partial<WaveView>;
    if (typeof held.panX !== 'number' || typeof held.pxPerSecond !== 'number') return;
    this.view = { panX: held.panX, pxPerSecond: held.pxPerSecond };
    this.needsFit = false;
    this.redraw();
  }

  fit(): void {
    const asset = this.asset();
    if (!asset || asset.duration <= 0 || this.w <= 0) {
      this.needsFit = true;
      return;
    }
    this.view = fitWave(asset.duration, this.w, FIT_MARGIN);
    this.needsFit = false;
    this.redraw();
  }

  get isPlaying(): boolean {
    return this.playing !== null;
  }

  /** Plays the picked clip, else the whole video, from its start, with the video's volume and loop. */
  play(): void {
    const asset = this.asset();
    const span = this.shownSpan();
    const element = asset && this.player(asset);
    if (!asset || !span || !element) return;
    this.playing = span;
    element.muted = false;
    element.volume = videoVolumeOf(asset.props);
    element.currentTime = span.start;
    void element.play().catch(() => {});
    this.invalidate();
  }

  stop(): void {
    this.playing = null;
    this.preview?.element.pause();
    this.invalidate();
  }

  /** Where the tab's player stands, in seconds of the video; null while nothing is shown. */
  get playhead(): number | null {
    return this.preview && this.preview.file === this.asset()?.file ? this.preview.element.currentTime : null;
  }

  /** Called every frame while the tab is on screen: the playhead follows the player, a clip loops. */
  tick(): void {
    const element = this.preview?.element;
    const span = this.playing;
    const asset = this.asset();
    if (!element || !span || !asset) return;
    if (element.currentTime >= span.end - LEAST_SPAN / 2 || element.ended) {
      if (videoLoopsOf(asset.props)) {
        element.currentTime = span.start;
        if (element.paused) void element.play().catch(() => {});
      } else {
        this.stop();
        this.onStopped();
      }
    }
    this.redraw();
  }

  /** The picked clip of the video shown, null for none. */
  pickedClip(): VideoClip | null {
    const asset = this.asset();
    const id = this.state.selectedClipId;
    return (asset && id && clipsOf<VideoClip>(asset).find((c) => c.id === id)) || null;
  }

  /** What the tab plays: a drag's span while one runs, else the picked clip, else the whole video. */
  shownSpan(): CutSpan | null {
    const asset = this.asset();
    if (!asset || asset.duration <= 0) return null;
    if (this.gesture?.kind === 'span' && this.gesture.moved) return this.gesture.span;
    const clip = this.pickedClip();
    return clip ? this.clipSpan(clip, asset) : { start: 0, end: asset.duration };
  }

  /** The middle of the strip at a second of the video, on screen (test hook). */
  stripAt(seconds: number): Vec2 {
    const origin = this.getGlobalPosition();
    return {
      x: origin.x + secondsToX(this.view, seconds),
      y: origin.y + this.timelineTop + RULER_HEIGHT + STRIP_HEIGHT / 2,
    };
  }

  /** The middle of the track at a second of the video, on screen (test hook). */
  trackAt(seconds: number): Vec2 {
    const origin = this.getGlobalPosition();
    return { x: origin.x + secondsToX(this.view, seconds), y: origin.y + this.h - TRACK_HEIGHT / 2 };
  }

  /**
   * Where the picture is drawn, whether the player has one, and the colour in the middle of the
   * frame the player holds, read from the video itself (test hook).
   */
  previewInfo(): {
    shown: boolean;
    x: number;
    y: number;
    w: number;
    h: number;
    time: number;
    paused: boolean;
    pixel: number[] | null;
  } {
    const p = this.picture.getGlobalPosition();
    return {
      pixel: this.preview ? middlePixel(this.preview.element) : null,
      shown: this.picture.visible,
      x: p.x,
      y: p.y,
      w: this.picture.width,
      h: this.picture.height,
      time: this.preview?.element.currentTime ?? 0,
      paused: this.preview?.element.paused ?? true,
    };
  }

  override destroy(): void {
    this.dropPreview();
    super.destroy({ children: true });
  }

  private get timelineTop(): number {
    return Math.max(0, this.h - TIMELINE_HEIGHT);
  }

  private asset(): VideoAsset | null {
    return (this.assetId && findAssetOf(this.state.project, this.assetId, 'video')) || null;
  }

  private clipSpan(clip: VideoClip, asset: VideoAsset): CutSpan {
    const { start, end } = videoSpanOf(clip.props);
    return spanOf(start, end, asset.duration);
  }

  private bands(asset: VideoAsset): ClipBand[] {
    const g = this.gesture?.kind === 'span' && this.gesture.moved ? this.gesture : null;
    const bands = clipsOf<VideoClip>(asset).map((clip) => ({
      id: clip.id,
      span: g?.clipId === clip.id ? g.span : this.clipSpan(clip, asset),
    }));
    return g && g.clipId === null ? [...bands, { id: '', span: g.span }] : bands;
  }

  /** The tab's player for the video's file, made anew when the file is another one. */
  private player(asset: VideoAsset): HTMLVideoElement | null {
    if (!asset.file) return null;
    if (this.preview?.file === asset.file) return this.preview.element;
    this.dropPreview();
    const held = this.shownSpan()?.start ?? 0;
    const preview: Preview = {
      file: asset.file,
      texture: null,
      element: openVideo(`/game-maker/data/videos/${asset.file}`, held, (texture) => {
        if (this.preview !== preview) return texture.destroy(true);
        preview.texture = texture;
        texture.source.on('update', () => this.invalidate());
        // It played for its first frame; until Play it stands at the moment the tab shows.
        if (!this.playing) {
          preview.element.pause();
          preview.element.currentTime = held;
        }
        this.invalidate();
      }),
    };
    preview.element.addEventListener('seeked', () => this.invalidate());
    this.preview = preview;
    return preview.element;
  }

  private dropPreview(): void {
    const preview = this.preview;
    if (!preview) return;
    this.preview = null;
    this.playing = null;
    this.picture.texture = Texture.EMPTY;
    preview.element.pause();
    preview.element.removeAttribute('src');
    preview.element.load();
    preview.texture?.destroy(true);
  }

  private seekTo(seconds: number): void {
    const asset = this.asset();
    const element = asset && this.player(asset);
    if (element) element.currentTime = Math.max(0, Math.min(asset.duration, seconds));
  }

  override onKeyDown(event: KeyboardEvent): boolean {
    if (event.code !== 'Space' || event.ctrlKey || event.metaKey || !this.asset()) return false;
    this.onTogglePlay();
    return true;
  }

  private onPointerDown(e: FederatedPointerEvent): void {
    const asset = this.asset();
    if (!asset || asset.duration <= 0) return;
    const p = this.toLocal(e.global);
    const y = p.y - this.timelineTop;
    if (y < 0) return;
    e.stopPropagation();
    ui().focus.focus(this);
    if (e.button === 1) {
      this.gesture = { kind: 'pan', pointerX: p.x, panX: this.view.panX };
      return;
    }
    if (e.button !== 0) return;
    const from = Math.max(0, Math.min(asset.duration, xToSeconds(this.view, p.x)));
    if (y < RULER_HEIGHT) {
      this.gesture = { kind: 'seek' };
      this.stop();
      this.seekTo(from);
      return;
    }
    const start = {
      kind: 'span',
      from,
      pointerX: p.x,
      moved: false,
      span: { start: from, end: from },
    } as const;
    if (y < RULER_HEIGHT + STRIP_HEIGHT) {
      this.gesture = { ...start, clipId: null, edge: null, picks: clipAt(this.view, this.bands(asset), p.x) };
      return;
    }
    const clip = this.pickedClip();
    const span = clip && this.clipSpan(clip, asset);
    const edge = span ? edgeAt(this.view, span, p.x) : null;
    const beside =
      span && !edge && (p.x < secondsToX(this.view, span.start) || p.x > secondsToX(this.view, span.end));
    this.gesture = { ...start, clipId: clip?.id ?? null, edge, picks: beside ? null : undefined };
  }

  private onPointerMove(e: FederatedPointerEvent): void {
    const p = this.toLocal(e.global);
    const g = this.gesture;
    const asset = this.asset();
    if (!g || !asset) return;
    if (g.kind === 'pan') {
      this.view = clampWave({ ...this.view, panX: g.panX + p.x - g.pointerX }, asset.duration, this.w);
      this.needsFit = false;
      this.redraw();
      return;
    }
    if (g.kind === 'seek') {
      this.seekTo(xToSeconds(this.view, p.x));
      return;
    }
    if (!g.moved && g.edge === null && Math.abs(p.x - g.pointerX) < DRAG_START) return;
    g.moved = true;
    const clip = this.pickedClip();
    const before = clip && clip.id === g.clipId ? this.clipSpan(clip, asset) : g.span;
    g.span = draggedSpan(before, g.edge, g.from, xToSeconds(this.view, p.x), asset.duration, LEAST_SPAN);
    this.seekTo(g.edge === 'end' ? g.span.end : g.span.start);
    this.redraw();
  }

  private onPointerUp(): void {
    const g = this.gesture;
    this.gesture = null;
    if (g?.kind !== 'span') return this.redraw();
    const asset = this.asset();
    if (!g.moved) {
      if (asset && g.picks !== undefined)
        this.state.select(
          g.picks
            ? { kind: 'clip', assetId: asset.id, clipId: g.picks }
            : { kind: 'asset', assetId: asset.id },
        );
      return this.redraw();
    }
    if (asset) this.commit(asset, g);
    this.redraw();
  }

  /** A drag's span as the clip's start and end in one step, or as a new clip, which is then picked. */
  private commit(asset: VideoAsset, g: Extract<Gesture, { kind: 'span' }>): void {
    const next = storedCut(g.span, asset.duration, STORED_RATE);
    if (g.clipId === null) {
      const add = new AddClipCommand(asset.id, { [VIDEO_START_KEY]: next.start, [VIDEO_END_KEY]: next.end });
      if (this.state.execute(add)) this.state.select({ kind: 'clip', assetId: asset.id, clipId: add.clipId });
      return;
    }
    const clip = clipsOf<VideoClip>(asset).find((c) => c.id === g.clipId);
    if (!clip) return;
    const now = videoSpanOf(clip.props);
    const values: PropertyBag = {};
    if (next.start !== now.start) values[VIDEO_START_KEY] = next.start;
    if (next.end !== now.end) values[VIDEO_END_KEY] = next.end;
    if (Object.keys(values).length > 0)
      this.state.execute(
        new SetClipPropsCommand(asset.id, clip.id, values, g.edge ? 'Move clip edge' : 'Span clip'),
      );
  }

  private onWheel(e: FederatedWheelEvent): void {
    const asset = this.asset();
    const p = this.toLocal(e.global);
    if (!asset || asset.duration <= 0 || p.y < this.timelineTop) return;
    this.view = clampWave(zoomWave(this.view, zoomFactor(e.deltaY), p.x), asset.duration, this.w);
    this.needsFit = false;
    this.redraw();
    e.preventDefault();
  }

  override layout(): void {
    if (this.needsFit) this.fit();
  }

  override redraw(): void {
    const asset = this.asset();
    if (this.needsFit && asset && asset.duration > 0 && this.w > 0) return this.fit();
    const c = Theme.colors;
    const top = this.timelineTop;
    this.bg.clear().rect(0, 0, this.w, this.h).fill(0x000000);
    this.bg.rect(0, top, this.w, RULER_HEIGHT + STRIP_HEIGHT).fill(c.panelHeader);
    this.bg.rect(0, top + RULER_HEIGHT - 1, this.w, 1).fill(c.border);
    this.bg.rect(0, top + RULER_HEIGHT + STRIP_HEIGHT - 1, this.w, 1).fill(c.border);
    this.bg.rect(0, top + RULER_HEIGHT + STRIP_HEIGHT, this.w, TRACK_HEIGHT).fill(c.viewport);
    this.overlay.clear();
    this.hint.visible = !asset?.file;
    this.hint.text = !asset ? 'No video selected – pick one in the Assets panel.' : 'Not uploaded.';
    this.hint.position.set(Theme.padding * 2, Theme.padding * 2);
    if (!asset?.file) {
      this.picture.visible = false;
      this.syncLabels([]);
      this.syncStrip([]);
      return;
    }
    this.drawPicture(asset, top);
    const left = Math.max(0, secondsToX(this.view, 0));
    const right = Math.min(this.w, secondsToX(this.view, asset.duration));
    this.drawRuler(left, right, top);
    this.drawStrip(asset, top);
    this.drawTrack(asset, left, right, top);
    const head = this.playhead;
    if (head !== null) {
      const x = secondsToX(this.view, head);
      if (x >= 0 && x <= this.w) this.overlay.rect(x - 0.5, top, 1, TIMELINE_HEIGHT).fill(c.hot);
    }
  }

  private drawPicture(asset: VideoAsset, top: number): void {
    this.player(asset);
    const texture = this.preview?.texture;
    this.picture.visible = Boolean(texture);
    if (!texture) return;
    const room = { w: this.w - PREVIEW_PAD * 2, h: top - PREVIEW_PAD * 2 };
    const fitted = containRect(asset.width || texture.width, asset.height || texture.height, room.w, room.h);
    this.picture.texture = texture;
    this.picture.position.set(PREVIEW_PAD + fitted.x, PREVIEW_PAD + fitted.y);
    this.picture.width = Math.max(1, fitted.w);
    this.picture.height = Math.max(1, fitted.h);
  }

  private drawRuler(left: number, right: number, top: number): void {
    const step = rulerStep(this.view.pxPerSecond, LABEL_SPACING);
    const placed: { text: string; x: number }[] = [];
    const first = Math.ceil(Math.max(0, xToSeconds(this.view, left)) / step);
    for (let k = first; ; k++) {
      const seconds = k * step;
      const x = secondsToX(this.view, seconds);
      if (x > right + 0.5) break;
      this.overlay.rect(x, top + RULER_HEIGHT - 8, 1, 7).fill({ color: Theme.colors.textMuted, alpha: 0.8 });
      placed.push({ text: rulerLabel(seconds, step), x: x + 3 });
    }
    this.syncLabels(placed.map((label) => ({ ...label, y: top + 6 })));
  }

  private litClipId(): string {
    const g = this.gesture?.kind === 'span' && this.gesture.moved ? this.gesture : null;
    return (g ? g.clipId : this.pickedClip()?.id) ?? '';
  }

  private drawStrip(asset: VideoAsset, top: number): void {
    const c = Theme.colors;
    const lit = this.litClipId();
    const names = new Map(clipsOf<VideoClip>(asset).map((clip) => [clip.id, clip.name]));
    const placed: { text: string; x: number; room: number; lit: boolean }[] = [];
    const y = top + RULER_HEIGHT;
    for (const { id, span } of this.bands(asset)) {
      const a = secondsToX(this.view, span.start);
      const b = secondsToX(this.view, span.end);
      if (b < 0 || a > this.w) continue;
      const on = id === lit;
      this.overlay
        .rect(a, y + 2, Math.max(1, b - a), STRIP_HEIGHT - 4)
        .fill({ color: c.accent, alpha: on ? 0.85 : 0.35 })
        .stroke({ width: 1, color: on ? c.accentText : c.accent, alpha: on ? 1 : 0.7, alignment: 1 });
      const name = names.get(id);
      if (name)
        placed.push({
          text: name,
          x: Math.max(a, 0) + 4,
          room: Math.min(b, this.w) - Math.max(a, 0) - 8,
          lit: on,
        });
    }
    this.syncStrip(placed.map((label) => ({ ...label, y })));
  }

  /** The video's length lit, what lies outside the picked clip dark, and a handle at each of its edges. */
  private drawTrack(asset: VideoAsset, left: number, right: number, top: number): void {
    const c = Theme.colors;
    const y = top + RULER_HEIGHT + STRIP_HEIGHT;
    const h = TRACK_HEIGHT;
    this.overlay.rect(left, y + 4, Math.max(0, right - left), h - 8).fill({ color: c.panel, alpha: 0.8 });
    const clip = this.pickedClip();
    const span =
      this.gesture?.kind === 'span' && this.gesture.moved
        ? this.gesture.span
        : clip && this.clipSpan(clip, asset);
    if (!span) return;
    const a = secondsToX(this.view, span.start);
    const b = secondsToX(this.view, span.end);
    this.overlay.rect(a, y + 4, Math.max(1, b - a), h - 8).fill({ color: c.accent, alpha: 0.35 });
    for (const [x, edge] of [
      [a, 'start'],
      [b, 'end'],
    ] as const) {
      const held = this.gesture?.kind === 'span' && this.gesture.edge === edge;
      this.overlay.rect(x - 1, y + 2, 2, h - 4).fill(held ? c.hot : c.accent);
      const toward = edge === 'start' ? 1 : -1;
      this.overlay
        .moveTo(x, y + 2)
        .lineTo(x + toward * 8, y + 2)
        .lineTo(x, y + 12)
        .closePath()
        .fill(held ? c.hot : c.accent);
    }
  }

  private syncLabels(placed: readonly { text: string; x: number; y: number }[]): void {
    placed.forEach((label, i) => {
      let text = this.texts[i];
      if (!text) {
        text = crispText({ style: textStyle('small') });
        this.texts.push(text);
        this.labels.addChild(text);
      }
      text.visible = true;
      if (text.text !== label.text) text.text = label.text;
      text.position.set(Math.round(label.x), Math.round(label.y));
    });
    for (let i = placed.length; i < this.texts.length; i++) this.texts[i]!.visible = false;
  }

  private syncStrip(
    placed: readonly { text: string; x: number; y: number; room: number; lit: boolean }[],
  ): void {
    placed.forEach((label, i) => {
      let text = this.stripTexts[i];
      if (!text) {
        text = crispText({ style: textStyle('small') });
        this.stripTexts.push(text);
        this.stripLabels.addChild(text);
      }
      const style = label.lit ? LIT_LABEL : textStyle('small');
      if (text.style !== style) text.style = style;
      if (text.text !== label.text) text.text = label.text;
      for (let cut = label.text.length; text.width > label.room && cut > 1; cut--)
        text.text = `${label.text.slice(0, cut - 1)}…`;
      text.visible = text.width <= label.room;
      text.position.set(Math.round(label.x), Math.round(label.y + (STRIP_HEIGHT - text.height) / 2));
    });
    for (let i = placed.length; i < this.stripTexts.length; i++) this.stripTexts[i]!.visible = false;
  }
}