feuerwerk/tests/core/composition.test.ts

/**
 * Compositions (#417): objects along a time axis. What lives at a frame, which clips play on it and
 * where its videos stand is `compositionAt`; `compositionHolder` writes it out for a view to draw.
 */
import { describe, expect, it } from 'vitest';
import {
  AddAnimationTrackCommand,
  SetAnimationKeyCommand,
  SetAnimationRootCommand,
  SetTrackRelativeCommand,
} from '../../src/core/commands/animationCommands';
import {
  CreateAssetCommand,
  CreateVideoCommand,
  DeleteAssetCommand,
  DuplicateAssetCommand,
} from '../../src/core/commands/assetCommands';
import { AddClipCommand } from '../../src/core/commands/clipCommands';
import type {
  AnimationAsset,
  CompositionAsset,
  CompositionItem,
  ContainerAsset,
} from '../../src/core/model/Asset';
import {
  clipLength,
  compositionAhead,
  compositionAt,
  compositionFps,
  compositionHolder,
  compositionLength,
  compositionLoops,
  fittedLength,
  itemVideo,
} from '../../src/core/model/composition';
import { instanceChildId, resolveObjects } from '../../src/core/model/instance';
import { findAssetOf, objectHolders } from '../../src/core/model/Project';
import type { PropertyBag } from '../../src/core/model/Properties';
import { COMPOSITION_LOOP_KEY } from '../../src/core/properties/compositions';
import { createSceneObject } from '../../src/core/model/SceneObject';
import {
  VIDEO_CLOCK_KEY,
  VIDEO_END_KEY,
  VIDEO_START_KEY,
  VIDEO_TIME_KEY,
} from '../../src/core/properties/videos';
import { makeProject, type ProjectFixture, roundTrip } from '../helpers';

const FILE = `a1b2c3d4e5f6/${'c7'.repeat(32)}.mp4`;

interface Cut extends ProjectFixture {
  compId: string;
  spriteId: string;
  comp(): CompositionAsset;
  /** Adds a track and returns its id. */
  track(name: string): string;
  /** Places `assetId` as an item on a track; returns the item's id, which is its object's. */
  item(
    track: string,
    assetId: string,
    start: number,
    length: number,
    extra?: Partial<CompositionItem>,
  ): string;
  animation(
    root: string,
    rows: [property: string, keys: [number, number][], target?: string][],
    fps?: number,
  ): string;
  clip(track: string, animation: string, start: number): string;
}

function makeCut(): Cut {
  const f = makeProject();
  const comp = new CreateAssetCommand('composition', 'Cut');
  const sprite = new CreateAssetCommand('sprite', 'Box');
  f.stack.execute(comp);
  f.stack.execute(sprite);
  const current = () => findAssetOf(f.project, comp.asset.id, 'composition')!;
  let next = 0;
  return {
    ...f,
    compId: comp.asset.id,
    spriteId: sprite.asset.id,
    comp: current,
    track(name) {
      const id = `t${++next}`;
      current().tracks.push({ id, name });
      return id;
    },
    item(track, assetId, start, length, extra = {}) {
      const id = `i${++next}`;
      current().objects.push(createSceneObject(id, assetId, { x: 0, y: 0 }));
      current().items.push({ id, track, start, length, offset: 0, ...extra });
      return id;
    },
    animation(root, rows, fps = 10) {
      const create = new CreateAssetCommand('animation', 'Move', { fps, playback: 'loop' });
      f.stack.execute(create);
      f.stack.execute(new SetAnimationRootCommand(create.asset.id, root));
      for (const [property, keys, target = ''] of rows) {
        const row = new AddAnimationTrackCommand(create.asset.id, target, property);
        f.stack.execute(row);
        for (const [frame, value] of keys)
          f.stack.execute(new SetAnimationKeyCommand(create.asset.id, row.trackId, frame, value));
      }
      return create.asset.id;
    },
    clip(track, animation, start) {
      const id = `c${++next}`;
      current().clips.push({ id, track, animation, start });
      return id;
    },
  };
}

const firstTrack = (cut: Cut) => cut.comp().tracks[0]!.id;
const liveIds = (cut: Cut, frame: number) =>
  compositionAt(cut.project, cut.comp(), frame).map((l) => l.object.id);

function addVideo(cut: Cut, duration = 10): string {
  const video = new CreateVideoCommand('Intro', FILE, {
    width: 320,
    height: 180,
    duration,
    fps: 30,
    audio: false,
  });
  cut.stack.execute(video);
  return video.asset.id;
}

function videoSprite(cut: Cut, texture: string): string {
  const sprite = new CreateAssetCommand('sprite', 'Screen', { texture });
  cut.stack.execute(sprite);
  return sprite.asset.id;
}

/** A container "Rig" holding a sprite "Arm", returned with the template id of the arm. */
function rig(cut: Cut, name = 'Rig', armProps: PropertyBag = {}): { id: string; arm: string } {
  const create = new CreateAssetCommand('container', name);
  cut.stack.execute(create);
  const arm = new CreateAssetCommand('sprite', 'Arm', armProps);
  cut.stack.execute(arm);
  const template = findAssetOf(cut.project, create.asset.id, 'container') as ContainerAsset;
  const armId = `arm-of-${name}`;
  template.objects.push(createSceneObject(armId, arm.asset.id, { x: 5, y: 0 }));
  return { id: create.asset.id, arm: armId };
}

describe('a new composition', () => {
  it('starts with one track, nothing on it, at 60 fps and not looping', () => {
    const cut = makeCut();
    const comp = cut.comp();
    expect(comp.tracks).toHaveLength(1);
    expect(comp.tracks[0]!.name).toBe('Track 1');
    expect([comp.objects, comp.items, comp.clips]).toEqual([[], [], []]);
    expect(compositionFps(comp)).toBe(60);
    expect(compositionLoops(comp)).toBe(false);
  });

  it('is an object holder, so deleting an asset takes the object it placed – and undo brings it back', () => {
    const cut = makeCut();
    const item = cut.item(firstTrack(cut), cut.spriteId, 0, 60);
    expect(objectHolders(cut.project).map((h) => h.id)).toContain(cut.compId);
    cut.stack.execute(new DeleteAssetCommand(cut.spriteId));
    expect(cut.comp().objects).toEqual([]);
    expect(liveIds(cut, 10)).toEqual([]);
    cut.stack.undo();
    expect(liveIds(cut, 10)).toEqual([item]);
  });

  it('a duplicate gets objects of its own, and its items follow them', () => {
    const cut = makeCut();
    cut.item(firstTrack(cut), cut.spriteId, 0, 60);
    const duplicate = new DuplicateAssetCommand(cut.compId);
    cut.stack.execute(duplicate);
    const copy = findAssetOf(cut.project, duplicate.copyId, 'composition')!;
    expect(copy.objects[0]!.id).not.toBe(cut.comp().objects[0]!.id);
    expect(copy.items[0]!.id).toBe(copy.objects[0]!.id);
    expect(compositionAt(cut.project, copy, 10).map((l) => l.object.id)).toEqual([copy.objects[0]!.id]);
  });
});

describe('what lives at a frame', () => {
  it('an item from its start up to, not including, its end', () => {
    const cut = makeCut();
    const item = cut.item(firstTrack(cut), cut.spriteId, 30, 60);
    expect(liveIds(cut, 29)).toEqual([]);
    expect(liveIds(cut, 30)).toEqual([item]);
    expect(liveIds(cut, 89)).toEqual([item]);
    expect(liveIds(cut, 90)).toEqual([]);
  });

  it('counts its time in seconds since its start, and its video time from its offset', () => {
    const cut = makeCut();
    cut.item(firstTrack(cut), cut.spriteId, 30, 120, { offset: 2 });
    const [live] = compositionAt(cut.project, cut.comp(), 60);
    expect(live!.time).toBe(0.5);
    expect(live!.videoTime).toBe(2.5);
  });

  it('nothing of an item whose object or track is gone', () => {
    const cut = makeCut();
    const kept = cut.item(firstTrack(cut), cut.spriteId, 0, 60);
    const orphan = cut.item(firstTrack(cut), cut.spriteId, 0, 60);
    cut.comp().objects = cut.comp().objects.filter((o) => o.id !== orphan);
    cut.item('no-such-track', cut.spriteId, 0, 60);
    expect(liveIds(cut, 10)).toEqual([kept]);
  });

  it('draws the last track first, so the first lies in front; a track by start', () => {
    const cut = makeCut();
    const top = firstTrack(cut);
    const bottom = cut.track('Background');
    const front = cut.item(top, cut.spriteId, 0, 60);
    const backLate = cut.item(bottom, cut.spriteId, 10, 60);
    const backEarly = cut.item(bottom, cut.spriteId, 0, 30);
    expect(liveIds(cut, 20)).toEqual([backEarly, backLate, front]);
    expect(
      resolveObjects(cut.project, compositionHolder(cut.project, cut.comp(), 20)).map((o) => o.id),
    ).toEqual([backEarly, backLate, front]);
  });

  it('lasts until the last item or clip ends', () => {
    const cut = makeCut();
    cut.item(firstTrack(cut), cut.spriteId, 0, 60);
    expect(compositionLength(cut.project, cut.comp())).toBe(60);
    cut.clip(
      firstTrack(cut),
      cut.animation(cut.spriteId, [
        [
          'x',
          [
            [0, 0],
            [9, 90],
          ],
        ],
      ]),
      40,
    );
    expect(compositionLength(cut.project, cut.comp())).toBe(100);
  });
});

describe('animation clips', () => {
  it('last one pass of their animation, counted in the frames of the composition', () => {
    const cut = makeCut();
    // Keys on 0…9 at 10 fps: ten frames, one second – 60 frames at 60 fps.
    const move = cut.animation(cut.spriteId, [
      [
        'x',
        [
          [0, 0],
          [9, 90],
        ],
      ],
    ]);
    const clip = cut.clip(firstTrack(cut), move, 0);
    expect(
      clipLength(
        cut.project,
        cut.comp(),
        cut.comp().clips.find((c) => c.id === clip)!,
      ),
    ).toBe(60);
  });

  it('play on the item of their track they overlap, from where the clip stands', () => {
    const cut = makeCut();
    const item = cut.item(firstTrack(cut), cut.spriteId, 0, 120);
    cut.clip(
      firstTrack(cut),
      cut.animation(cut.spriteId, [
        [
          'x',
          [
            [0, 0],
            [9, 90],
          ],
        ],
      ]),
      30,
    );
    const x = (frame: number) =>
      compositionHolder(cut.project, cut.comp(), frame).objects.find((o) => o.id === item)!.position.x;
    expect(x(29)).toBe(0);
    expect(x(30)).toBe(0);
    // 30 frames at 60 fps after its start: 0.5 s, frame 5 of the animation.
    expect(x(60)).toBe(50);
    expect(compositionAt(cut.project, cut.comp(), 60)[0]!.clips[0]!.frame).toBe(5);
  });

  it('hold their last frame once through, for as long as the item lasts', () => {
    const cut = makeCut();
    const item = cut.item(firstTrack(cut), cut.spriteId, 0, 300);
    cut.clip(
      firstTrack(cut),
      cut.animation(cut.spriteId, [
        [
          'x',
          [
            [0, 0],
            [9, 90],
          ],
        ],
      ]),
      0,
    );
    const holder = compositionHolder(cut.project, cut.comp(), 200);
    expect(holder.objects.find((o) => o.id === item)!.position.x).toBe(90);
  });

  it('play on neither an item of another track nor one that starts after they ended', () => {
    const cut = makeCut();
    const other = cut.track('Other');
    const elsewhere = cut.item(other, cut.spriteId, 0, 300);
    const later = cut.item(firstTrack(cut), cut.spriteId, 100, 60);
    cut.clip(
      firstTrack(cut),
      cut.animation(cut.spriteId, [
        [
          'x',
          [
            [0, 10],
            [9, 90],
          ],
        ],
      ]),
      0,
    );
    const at = compositionAt(cut.project, cut.comp(), 120);
    expect(at.find((l) => l.object.id === elsewhere)!.clips).toEqual([]);
    expect(at.find((l) => l.object.id === later)!.clips).toEqual([]);
  });

  it('running over two items, play on each in turn', () => {
    const cut = makeCut();
    const track = firstTrack(cut);
    const first = cut.item(track, cut.spriteId, 0, 30);
    const second = cut.item(track, cut.spriteId, 30, 60);
    cut.clip(
      track,
      cut.animation(cut.spriteId, [
        [
          'x',
          [
            [0, 0],
            [9, 90],
          ],
        ],
      ]),
      0,
    );
    const x = (frame: number, id: string) =>
      compositionHolder(cut.project, cut.comp(), frame).objects.find((o) => o.id === id)!.position.x;
    expect(x(12, first)).toBe(20);
    expect(x(42, second)).toBe(70);
  });

  it('over two items of a track that overlap, play on both – the later one in front', () => {
    const cut = makeCut();
    const track = firstTrack(cut);
    const early = cut.item(track, cut.spriteId, 0, 60);
    const late = cut.item(track, cut.spriteId, 30, 60);
    cut.clip(
      track,
      cut.animation(cut.spriteId, [
        [
          'x',
          [
            [0, 0],
            [9, 90],
          ],
        ],
      ]),
      0,
    );
    expect(liveIds(cut, 42)).toEqual([early, late]);
    const holder = compositionHolder(cut.project, cut.comp(), 42);
    expect([early, late].map((id) => holder.objects.find((o) => o.id === id)!.position.x)).toEqual([70, 70]);
  });

  it('add a relative row to the value the item has', () => {
    const cut = makeCut();
    const item = cut.item(firstTrack(cut), cut.spriteId, 0, 120);
    cut.comp().objects[0]!.position.x = 100;
    const move = cut.animation(cut.spriteId, [
      [
        'x',
        [
          [0, 0],
          [9, 90],
        ],
      ],
    ]);
    const animation = findAssetOf(cut.project, move, 'animation') as AnimationAsset;
    cut.stack.execute(new SetTrackRelativeCommand(move, animation.tracks[0]!.id, true));
    cut.clip(firstTrack(cut), move, 0);
    const holder = compositionHolder(cut.project, cut.comp(), 30);
    expect(holder.objects.find((o) => o.id === item)!.position.x).toBe(150);
  });

  it('reach into a container item by template path, and by names when built over another', () => {
    const cut = makeCut();
    const own = rig(cut);
    const lookalike = rig(cut, 'Other rig');
    const item = cut.item(firstTrack(cut), lookalike.id, 0, 120);
    // Built over "Rig": its row names Rig's arm, which "Other rig" has by name only.
    cut.clip(
      firstTrack(cut),
      cut.animation(own.id, [
        [
          'rotation',
          [
            [0, 0],
            [9, 90],
          ],
          own.arm,
        ],
      ]),
      0,
    );
    const holder = compositionHolder(cut.project, cut.comp(), 30);
    const arm = resolveObjects(cut.project, holder).find(
      (o) => o.id === instanceChildId(item, lookalike.arm),
    )!;
    expect(arm.overrides.rotation).toBe(50);

    const direct = makeCut();
    const same = rig(direct);
    const placed = direct.item(firstTrack(direct), same.id, 0, 120);
    direct.clip(
      firstTrack(direct),
      direct.animation(same.id, [
        [
          'rotation',
          [
            [0, 0],
            [9, 90],
          ],
          same.arm,
        ],
      ]),
      0,
    );
    const drawn = resolveObjects(direct.project, compositionHolder(direct.project, direct.comp(), 30));
    expect(drawn.find((o) => o.id === instanceChildId(placed, same.arm))!.overrides.rotation).toBe(50);
  });

  it('of an animation over a scene play nothing, and one gone neither', () => {
    const cut = makeCut();
    const scene = new CreateAssetCommand('scene', 'Level');
    cut.stack.execute(scene);
    cut.item(firstTrack(cut), cut.spriteId, 0, 120);
    const overScene = cut.animation(scene.asset.id, [
      [
        'gravityY',
        [
          [0, 0],
          [9, 90],
        ],
      ],
    ]);
    cut.clip(firstTrack(cut), overScene, 0);
    cut.clip(firstTrack(cut), 'gone', 0);
    expect(cut.comp().clips.map((c) => clipLength(cut.project, cut.comp(), c))).toEqual([0, 0]);
    expect(compositionAt(cut.project, cut.comp(), 10)[0]!.clips).toEqual([]);
  });

  it('leave the document alone', () => {
    const cut = makeCut();
    cut.item(firstTrack(cut), cut.spriteId, 0, 120);
    cut.clip(
      firstTrack(cut),
      cut.animation(cut.spriteId, [
        [
          'x',
          [
            [0, 0],
            [9, 90],
          ],
        ],
      ]),
      0,
    );
    const before = structuredClone(cut.comp());
    compositionHolder(cut.project, cut.comp(), 30);
    expect(cut.comp()).toEqual(before);
  });
});

describe('items wearing a video', () => {
  it('drive it at the item’s video time', () => {
    const cut = makeCut();
    const screen = videoSprite(cut, addVideo(cut));
    const item = cut.item(firstTrack(cut), screen, 60, 600, { offset: 1.5 });
    const object = compositionHolder(cut.project, cut.comp(), 90).objects.find((o) => o.id === item)!;
    expect(object.overrides[VIDEO_CLOCK_KEY]).toBe('driven');
    expect(object.overrides[VIDEO_TIME_KEY]).toBe(2);
  });

  it('drive one worn inside a container item too – unless that one is driven by hand', () => {
    const cut = makeCut();
    const video = addVideo(cut);
    const free = rig(cut, 'Free', { texture: video });
    const byHand = rig(cut, 'Hand', { texture: video, [VIDEO_CLOCK_KEY]: 'driven', [VIDEO_TIME_KEY]: 7 });
    const a = cut.item(firstTrack(cut), free.id, 0, 120);
    const b = cut.item(cut.track('B'), byHand.id, 0, 120);
    const drawn = resolveObjects(cut.project, compositionHolder(cut.project, cut.comp(), 60));
    const arm = (item: string, template: string) =>
      drawn.find((o) => o.id === instanceChildId(item, template))!;
    expect(arm(a, free.arm).overrides[VIDEO_TIME_KEY]).toBe(1);
    expect(arm(a, free.arm).overrides[VIDEO_CLOCK_KEY]).toBe('driven');
    // Its own 7 s stay: nothing is written over the sprite's driven time.
    expect(arm(b, byHand.arm).overrides).toEqual({});
  });

  it('take the length of the video, or of the clip, less the offset when fitted', () => {
    const cut = makeCut();
    const video = addVideo(cut, 10);
    const clip = new AddClipCommand(video, { [VIDEO_START_KEY]: 0.5, [VIDEO_END_KEY]: 2 });
    cut.stack.execute(clip);
    const track = firstTrack(cut);
    const whole = cut.item(track, videoSprite(cut, video), 0, 1, { fitMedia: true });
    const trimmed = cut.item(track, videoSprite(cut, video), 0, 1, { offset: 2 });
    const short = cut.item(track, videoSprite(cut, clip.clipId), 0, 1);
    const plain = cut.item(track, cut.spriteId, 0, 1);
    const fitted = (id: string) =>
      fittedLength(
        cut.project,
        cut.comp(),
        cut.comp().items.find((i) => i.id === id)!,
      );
    expect(fitted(whole)).toBe(600);
    expect(fitted(trimmed)).toBe(480);
    expect(fitted(short)).toBe(90);
    expect(fitted(plain)).toBe(180);
    expect(
      itemVideo(
        cut.project,
        cut.comp(),
        cut.comp().items.find((i) => i.id === plain)!,
      ),
    ).toBeNull();
  });

  it('take the default length while the video’s length is not known yet', () => {
    const cut = makeCut();
    const item = cut.item(firstTrack(cut), videoSprite(cut, addVideo(cut, 0)), 0, 1);
    expect(
      fittedLength(
        cut.project,
        cut.comp(),
        cut.comp().items.find((i) => i.id === item)!,
      ),
    ).toBe(180);
  });

  it('are written out a second before they begin, at their first frame, so a view opens them ahead', () => {
    const cut = makeCut();
    const screen = videoSprite(cut, addVideo(cut));
    const track = firstTrack(cut);
    cut.item(track, screen, 0, 120);
    const next = cut.item(track, screen, 120, 120, { offset: 2 });
    const ahead = (frame: number) => compositionAhead(cut.project, cut.comp(), frame).objects;
    expect(ahead(59)).toEqual([]);
    const [object] = ahead(60);
    expect(object!.id).toBe(next);
    expect(object!.overrides[VIDEO_CLOCK_KEY]).toBe('driven');
    expect(object!.overrides[VIDEO_TIME_KEY]).toBe(2);
    // Begun, it wears its own time and is no longer ahead.
    expect(ahead(120)).toEqual([]);
  });

  it('at the start are ahead too when it loops and its end comes close – unless still alive', () => {
    const cut = makeCut();
    const screen = videoSprite(cut, addVideo(cut));
    const track = firstTrack(cut);
    const first = cut.item(track, screen, 0, 120);
    cut.item(track, screen, 120, 120);
    const ids = (frame: number) => compositionAhead(cut.project, cut.comp(), frame).objects.map((o) => o.id);
    expect(ids(200)).toEqual([]);
    cut.comp().props[COMPOSITION_LOOP_KEY] = true;
    expect(ids(179)).toEqual([]);
    expect(ids(200)).toEqual([first]);
    cut.comp().items[0]!.length = 240;
    cut.comp().items.pop();
    expect(ids(200)).toEqual([]);
  });

  it('follow the texture an override gives the item', () => {
    const cut = makeCut();
    const item = cut.item(firstTrack(cut), cut.spriteId, 0, 1);
    cut.comp().objects[0]!.overrides.texture = addVideo(cut, 4);
    expect(
      fittedLength(
        cut.project,
        cut.comp(),
        cut.comp().items.find((i) => i.id === item)!,
      ),
    ).toBe(240);
  });
});

describe('saving and loading a composition', () => {
  it('keeps tracks, items, clips and the objects', () => {
    const cut = makeCut();
    const track = cut.track('Second');
    cut.item(track, cut.spriteId, 12, 34, { offset: 1.25, fitMedia: true });
    cut.clip(track, 'gone', 5);
    const loaded = findAssetOf(roundTrip(cut.project), cut.compId, 'composition')!;
    expect(loaded.tracks).toEqual(cut.comp().tracks);
    expect(loaded.items).toEqual(cut.comp().items);
    expect(loaded.clips).toEqual(cut.comp().clips);
    expect(loaded.objects).toEqual(cut.comp().objects);
  });

  it('drops what no item spawns, an item without its object or track, and what is no time', () => {
    const cut = makeCut();
    const track = firstTrack(cut);
    const kept = cut.item(track, cut.spriteId, 0, 10);
    cut.comp().objects.push(createSceneObject('stray', cut.spriteId, { x: 0, y: 0 }));
    cut.comp().items.push({ id: 'no-object', track, start: 0, length: 10, offset: 0 });
    cut.item('no-track', cut.spriteId, 0, 10);
    cut.item(track, cut.spriteId, -5, 10);
    cut.item(track, cut.spriteId, 0, 0);
    cut.clip('no-track', 'x', 0);
    cut.clip(track, 'y', -1);
    const loaded = findAssetOf(roundTrip(cut.project), cut.compId, 'composition')!;
    expect(loaded.items.map((i) => i.id)).toEqual([kept]);
    expect(loaded.objects.map((o) => o.id)).toEqual([kept]);
    expect(loaded.clips).toEqual([]);
  });

  it('keeps every object at the top level, a negative offset, and at least one track', () => {
    const cut = makeCut();
    const track = firstTrack(cut);
    const parent = cut.item(track, cut.spriteId, 0, 10);
    const child = cut.item(track, cut.spriteId, 10, 10, { offset: -3 });
    cut.comp().objects.find((o) => o.id === child)!.parentId = parent;
    const raw = structuredClone(cut.project);
    findAssetOf(raw, cut.compId, 'composition')!.tracks = [];
    const loaded = findAssetOf(roundTrip(raw), cut.compId, 'composition')!;
    expect(loaded.tracks).toHaveLength(1);
    // The one track left is a new one, so the items on the old one fall away.
    expect(loaded.items).toEqual([]);
    const again = findAssetOf(roundTrip(cut.project), cut.compId, 'composition')!;
    expect(again.objects.map((o) => o.parentId)).toEqual(['', '']);
    expect(again.items.find((i) => i.id === child)!.offset).toBe(-3);
    expect(again.items.some((i) => 'fitMedia' in i)).toBe(false);
  });
});