/** * Compositions (#417): objects along a time axis. What lives at a frame, which clips play on it and * where its videos stand is `compositionAt`; `compositionHolder` writes it out for a view to draw. */ import { describe, expect, it } from 'vitest'; import { AddAnimationTrackCommand, SetAnimationKeyCommand, SetAnimationRootCommand, SetTrackRelativeCommand, } from '../../src/core/commands/animationCommands'; import { CreateAssetCommand, CreateVideoCommand, DeleteAssetCommand, DuplicateAssetCommand, } from '../../src/core/commands/assetCommands'; import { AddClipCommand } from '../../src/core/commands/clipCommands'; import type { AnimationAsset, CompositionAsset, CompositionItem, ContainerAsset, } from '../../src/core/model/Asset'; import { clipLength, compositionAhead, compositionAt, compositionFps, compositionHolder, compositionLength, compositionLoops, fittedLength, itemVideo, } from '../../src/core/model/composition'; import { instanceChildId, resolveObjects } from '../../src/core/model/instance'; import { findAssetOf, objectHolders } from '../../src/core/model/Project'; import type { PropertyBag } from '../../src/core/model/Properties'; import { COMPOSITION_LOOP_KEY } from '../../src/core/properties/compositions'; import { createSceneObject } from '../../src/core/model/SceneObject'; import { VIDEO_CLOCK_KEY, VIDEO_END_KEY, VIDEO_START_KEY, VIDEO_TIME_KEY, } from '../../src/core/properties/videos'; import { makeProject, type ProjectFixture, roundTrip } from '../helpers'; const FILE = `a1b2c3d4e5f6/${'c7'.repeat(32)}.mp4`; interface Cut extends ProjectFixture { compId: string; spriteId: string; comp(): CompositionAsset; /** Adds a track and returns its id. */ track(name: string): string; /** Places `assetId` as an item on a track; returns the item's id, which is its object's. */ item( track: string, assetId: string, start: number, length: number, extra?: Partial, ): string; animation( root: string, rows: [property: string, keys: [number, number][], target?: string][], fps?: number, ): string; clip(track: string, animation: string, start: number): string; } function makeCut(): Cut { const f = makeProject(); const comp = new CreateAssetCommand('composition', 'Cut'); const sprite = new CreateAssetCommand('sprite', 'Box'); f.stack.execute(comp); f.stack.execute(sprite); const current = () => findAssetOf(f.project, comp.asset.id, 'composition')!; let next = 0; return { ...f, compId: comp.asset.id, spriteId: sprite.asset.id, comp: current, track(name) { const id = `t${++next}`; current().tracks.push({ id, name }); return id; }, item(track, assetId, start, length, extra = {}) { const id = `i${++next}`; current().objects.push(createSceneObject(id, assetId, { x: 0, y: 0 })); current().items.push({ id, track, start, length, offset: 0, ...extra }); return id; }, animation(root, rows, fps = 10) { const create = new CreateAssetCommand('animation', 'Move', { fps, playback: 'loop' }); f.stack.execute(create); f.stack.execute(new SetAnimationRootCommand(create.asset.id, root)); for (const [property, keys, target = ''] of rows) { const row = new AddAnimationTrackCommand(create.asset.id, target, property); f.stack.execute(row); for (const [frame, value] of keys) f.stack.execute(new SetAnimationKeyCommand(create.asset.id, row.trackId, frame, value)); } return create.asset.id; }, clip(track, animation, start) { const id = `c${++next}`; current().clips.push({ id, track, animation, start }); return id; }, }; } const firstTrack = (cut: Cut) => cut.comp().tracks[0]!.id; const liveIds = (cut: Cut, frame: number) => compositionAt(cut.project, cut.comp(), frame).map((l) => l.object.id); function addVideo(cut: Cut, duration = 10): string { const video = new CreateVideoCommand('Intro', FILE, { width: 320, height: 180, duration, fps: 30, audio: false, }); cut.stack.execute(video); return video.asset.id; } function videoSprite(cut: Cut, texture: string): string { const sprite = new CreateAssetCommand('sprite', 'Screen', { texture }); cut.stack.execute(sprite); return sprite.asset.id; } /** A container "Rig" holding a sprite "Arm", returned with the template id of the arm. */ function rig(cut: Cut, name = 'Rig', armProps: PropertyBag = {}): { id: string; arm: string } { const create = new CreateAssetCommand('container', name); cut.stack.execute(create); const arm = new CreateAssetCommand('sprite', 'Arm', armProps); cut.stack.execute(arm); const template = findAssetOf(cut.project, create.asset.id, 'container') as ContainerAsset; const armId = `arm-of-${name}`; template.objects.push(createSceneObject(armId, arm.asset.id, { x: 5, y: 0 })); return { id: create.asset.id, arm: armId }; } describe('a new composition', () => { it('starts with one track, nothing on it, at 60 fps and not looping', () => { const cut = makeCut(); const comp = cut.comp(); expect(comp.tracks).toHaveLength(1); expect(comp.tracks[0]!.name).toBe('Track 1'); expect([comp.objects, comp.items, comp.clips]).toEqual([[], [], []]); expect(compositionFps(comp)).toBe(60); expect(compositionLoops(comp)).toBe(false); }); it('is an object holder, so deleting an asset takes the object it placed – and undo brings it back', () => { const cut = makeCut(); const item = cut.item(firstTrack(cut), cut.spriteId, 0, 60); expect(objectHolders(cut.project).map((h) => h.id)).toContain(cut.compId); cut.stack.execute(new DeleteAssetCommand(cut.spriteId)); expect(cut.comp().objects).toEqual([]); expect(liveIds(cut, 10)).toEqual([]); cut.stack.undo(); expect(liveIds(cut, 10)).toEqual([item]); }); it('a duplicate gets objects of its own, and its items follow them', () => { const cut = makeCut(); cut.item(firstTrack(cut), cut.spriteId, 0, 60); const duplicate = new DuplicateAssetCommand(cut.compId); cut.stack.execute(duplicate); const copy = findAssetOf(cut.project, duplicate.copyId, 'composition')!; expect(copy.objects[0]!.id).not.toBe(cut.comp().objects[0]!.id); expect(copy.items[0]!.id).toBe(copy.objects[0]!.id); expect(compositionAt(cut.project, copy, 10).map((l) => l.object.id)).toEqual([copy.objects[0]!.id]); }); }); describe('what lives at a frame', () => { it('an item from its start up to, not including, its end', () => { const cut = makeCut(); const item = cut.item(firstTrack(cut), cut.spriteId, 30, 60); expect(liveIds(cut, 29)).toEqual([]); expect(liveIds(cut, 30)).toEqual([item]); expect(liveIds(cut, 89)).toEqual([item]); expect(liveIds(cut, 90)).toEqual([]); }); it('counts its time in seconds since its start, and its video time from its offset', () => { const cut = makeCut(); cut.item(firstTrack(cut), cut.spriteId, 30, 120, { offset: 2 }); const [live] = compositionAt(cut.project, cut.comp(), 60); expect(live!.time).toBe(0.5); expect(live!.videoTime).toBe(2.5); }); it('nothing of an item whose object or track is gone', () => { const cut = makeCut(); const kept = cut.item(firstTrack(cut), cut.spriteId, 0, 60); const orphan = cut.item(firstTrack(cut), cut.spriteId, 0, 60); cut.comp().objects = cut.comp().objects.filter((o) => o.id !== orphan); cut.item('no-such-track', cut.spriteId, 0, 60); expect(liveIds(cut, 10)).toEqual([kept]); }); it('draws the last track first, so the first lies in front; a track by start', () => { const cut = makeCut(); const top = firstTrack(cut); const bottom = cut.track('Background'); const front = cut.item(top, cut.spriteId, 0, 60); const backLate = cut.item(bottom, cut.spriteId, 10, 60); const backEarly = cut.item(bottom, cut.spriteId, 0, 30); expect(liveIds(cut, 20)).toEqual([backEarly, backLate, front]); expect( resolveObjects(cut.project, compositionHolder(cut.project, cut.comp(), 20)).map((o) => o.id), ).toEqual([backEarly, backLate, front]); }); it('lasts until the last item or clip ends', () => { const cut = makeCut(); cut.item(firstTrack(cut), cut.spriteId, 0, 60); expect(compositionLength(cut.project, cut.comp())).toBe(60); cut.clip( firstTrack(cut), cut.animation(cut.spriteId, [ [ 'x', [ [0, 0], [9, 90], ], ], ]), 40, ); expect(compositionLength(cut.project, cut.comp())).toBe(100); }); }); describe('animation clips', () => { it('last one pass of their animation, counted in the frames of the composition', () => { const cut = makeCut(); // Keys on 0…9 at 10 fps: ten frames, one second – 60 frames at 60 fps. const move = cut.animation(cut.spriteId, [ [ 'x', [ [0, 0], [9, 90], ], ], ]); const clip = cut.clip(firstTrack(cut), move, 0); expect( clipLength( cut.project, cut.comp(), cut.comp().clips.find((c) => c.id === clip)!, ), ).toBe(60); }); it('play on the item of their track they overlap, from where the clip stands', () => { const cut = makeCut(); const item = cut.item(firstTrack(cut), cut.spriteId, 0, 120); cut.clip( firstTrack(cut), cut.animation(cut.spriteId, [ [ 'x', [ [0, 0], [9, 90], ], ], ]), 30, ); const x = (frame: number) => compositionHolder(cut.project, cut.comp(), frame).objects.find((o) => o.id === item)!.position.x; expect(x(29)).toBe(0); expect(x(30)).toBe(0); // 30 frames at 60 fps after its start: 0.5 s, frame 5 of the animation. expect(x(60)).toBe(50); expect(compositionAt(cut.project, cut.comp(), 60)[0]!.clips[0]!.frame).toBe(5); }); it('hold their last frame once through, for as long as the item lasts', () => { const cut = makeCut(); const item = cut.item(firstTrack(cut), cut.spriteId, 0, 300); cut.clip( firstTrack(cut), cut.animation(cut.spriteId, [ [ 'x', [ [0, 0], [9, 90], ], ], ]), 0, ); const holder = compositionHolder(cut.project, cut.comp(), 200); expect(holder.objects.find((o) => o.id === item)!.position.x).toBe(90); }); it('play on neither an item of another track nor one that starts after they ended', () => { const cut = makeCut(); const other = cut.track('Other'); const elsewhere = cut.item(other, cut.spriteId, 0, 300); const later = cut.item(firstTrack(cut), cut.spriteId, 100, 60); cut.clip( firstTrack(cut), cut.animation(cut.spriteId, [ [ 'x', [ [0, 10], [9, 90], ], ], ]), 0, ); const at = compositionAt(cut.project, cut.comp(), 120); expect(at.find((l) => l.object.id === elsewhere)!.clips).toEqual([]); expect(at.find((l) => l.object.id === later)!.clips).toEqual([]); }); it('running over two items, play on each in turn', () => { const cut = makeCut(); const track = firstTrack(cut); const first = cut.item(track, cut.spriteId, 0, 30); const second = cut.item(track, cut.spriteId, 30, 60); cut.clip( track, cut.animation(cut.spriteId, [ [ 'x', [ [0, 0], [9, 90], ], ], ]), 0, ); const x = (frame: number, id: string) => compositionHolder(cut.project, cut.comp(), frame).objects.find((o) => o.id === id)!.position.x; expect(x(12, first)).toBe(20); expect(x(42, second)).toBe(70); }); it('over two items of a track that overlap, play on both – the later one in front', () => { const cut = makeCut(); const track = firstTrack(cut); const early = cut.item(track, cut.spriteId, 0, 60); const late = cut.item(track, cut.spriteId, 30, 60); cut.clip( track, cut.animation(cut.spriteId, [ [ 'x', [ [0, 0], [9, 90], ], ], ]), 0, ); expect(liveIds(cut, 42)).toEqual([early, late]); const holder = compositionHolder(cut.project, cut.comp(), 42); expect([early, late].map((id) => holder.objects.find((o) => o.id === id)!.position.x)).toEqual([70, 70]); }); it('add a relative row to the value the item has', () => { const cut = makeCut(); const item = cut.item(firstTrack(cut), cut.spriteId, 0, 120); cut.comp().objects[0]!.position.x = 100; const move = cut.animation(cut.spriteId, [ [ 'x', [ [0, 0], [9, 90], ], ], ]); const animation = findAssetOf(cut.project, move, 'animation') as AnimationAsset; cut.stack.execute(new SetTrackRelativeCommand(move, animation.tracks[0]!.id, true)); cut.clip(firstTrack(cut), move, 0); const holder = compositionHolder(cut.project, cut.comp(), 30); expect(holder.objects.find((o) => o.id === item)!.position.x).toBe(150); }); it('reach into a container item by template path, and by names when built over another', () => { const cut = makeCut(); const own = rig(cut); const lookalike = rig(cut, 'Other rig'); const item = cut.item(firstTrack(cut), lookalike.id, 0, 120); // Built over "Rig": its row names Rig's arm, which "Other rig" has by name only. cut.clip( firstTrack(cut), cut.animation(own.id, [ [ 'rotation', [ [0, 0], [9, 90], ], own.arm, ], ]), 0, ); const holder = compositionHolder(cut.project, cut.comp(), 30); const arm = resolveObjects(cut.project, holder).find( (o) => o.id === instanceChildId(item, lookalike.arm), )!; expect(arm.overrides.rotation).toBe(50); const direct = makeCut(); const same = rig(direct); const placed = direct.item(firstTrack(direct), same.id, 0, 120); direct.clip( firstTrack(direct), direct.animation(same.id, [ [ 'rotation', [ [0, 0], [9, 90], ], same.arm, ], ]), 0, ); const drawn = resolveObjects(direct.project, compositionHolder(direct.project, direct.comp(), 30)); expect(drawn.find((o) => o.id === instanceChildId(placed, same.arm))!.overrides.rotation).toBe(50); }); it('of an animation over a scene play nothing, and one gone neither', () => { const cut = makeCut(); const scene = new CreateAssetCommand('scene', 'Level'); cut.stack.execute(scene); cut.item(firstTrack(cut), cut.spriteId, 0, 120); const overScene = cut.animation(scene.asset.id, [ [ 'gravityY', [ [0, 0], [9, 90], ], ], ]); cut.clip(firstTrack(cut), overScene, 0); cut.clip(firstTrack(cut), 'gone', 0); expect(cut.comp().clips.map((c) => clipLength(cut.project, cut.comp(), c))).toEqual([0, 0]); expect(compositionAt(cut.project, cut.comp(), 10)[0]!.clips).toEqual([]); }); it('leave the document alone', () => { const cut = makeCut(); cut.item(firstTrack(cut), cut.spriteId, 0, 120); cut.clip( firstTrack(cut), cut.animation(cut.spriteId, [ [ 'x', [ [0, 0], [9, 90], ], ], ]), 0, ); const before = structuredClone(cut.comp()); compositionHolder(cut.project, cut.comp(), 30); expect(cut.comp()).toEqual(before); }); }); describe('items wearing a video', () => { it('drive it at the item’s video time', () => { const cut = makeCut(); const screen = videoSprite(cut, addVideo(cut)); const item = cut.item(firstTrack(cut), screen, 60, 600, { offset: 1.5 }); const object = compositionHolder(cut.project, cut.comp(), 90).objects.find((o) => o.id === item)!; expect(object.overrides[VIDEO_CLOCK_KEY]).toBe('driven'); expect(object.overrides[VIDEO_TIME_KEY]).toBe(2); }); it('drive one worn inside a container item too – unless that one is driven by hand', () => { const cut = makeCut(); const video = addVideo(cut); const free = rig(cut, 'Free', { texture: video }); const byHand = rig(cut, 'Hand', { texture: video, [VIDEO_CLOCK_KEY]: 'driven', [VIDEO_TIME_KEY]: 7 }); const a = cut.item(firstTrack(cut), free.id, 0, 120); const b = cut.item(cut.track('B'), byHand.id, 0, 120); const drawn = resolveObjects(cut.project, compositionHolder(cut.project, cut.comp(), 60)); const arm = (item: string, template: string) => drawn.find((o) => o.id === instanceChildId(item, template))!; expect(arm(a, free.arm).overrides[VIDEO_TIME_KEY]).toBe(1); expect(arm(a, free.arm).overrides[VIDEO_CLOCK_KEY]).toBe('driven'); // Its own 7 s stay: nothing is written over the sprite's driven time. expect(arm(b, byHand.arm).overrides).toEqual({}); }); it('take the length of the video, or of the clip, less the offset when fitted', () => { const cut = makeCut(); const video = addVideo(cut, 10); const clip = new AddClipCommand(video, { [VIDEO_START_KEY]: 0.5, [VIDEO_END_KEY]: 2 }); cut.stack.execute(clip); const track = firstTrack(cut); const whole = cut.item(track, videoSprite(cut, video), 0, 1, { fitMedia: true }); const trimmed = cut.item(track, videoSprite(cut, video), 0, 1, { offset: 2 }); const short = cut.item(track, videoSprite(cut, clip.clipId), 0, 1); const plain = cut.item(track, cut.spriteId, 0, 1); const fitted = (id: string) => fittedLength( cut.project, cut.comp(), cut.comp().items.find((i) => i.id === id)!, ); expect(fitted(whole)).toBe(600); expect(fitted(trimmed)).toBe(480); expect(fitted(short)).toBe(90); expect(fitted(plain)).toBe(180); expect( itemVideo( cut.project, cut.comp(), cut.comp().items.find((i) => i.id === plain)!, ), ).toBeNull(); }); it('take the default length while the video’s length is not known yet', () => { const cut = makeCut(); const item = cut.item(firstTrack(cut), videoSprite(cut, addVideo(cut, 0)), 0, 1); expect( fittedLength( cut.project, cut.comp(), cut.comp().items.find((i) => i.id === item)!, ), ).toBe(180); }); it('are written out a second before they begin, at their first frame, so a view opens them ahead', () => { const cut = makeCut(); const screen = videoSprite(cut, addVideo(cut)); const track = firstTrack(cut); cut.item(track, screen, 0, 120); const next = cut.item(track, screen, 120, 120, { offset: 2 }); const ahead = (frame: number) => compositionAhead(cut.project, cut.comp(), frame).objects; expect(ahead(59)).toEqual([]); const [object] = ahead(60); expect(object!.id).toBe(next); expect(object!.overrides[VIDEO_CLOCK_KEY]).toBe('driven'); expect(object!.overrides[VIDEO_TIME_KEY]).toBe(2); // Begun, it wears its own time and is no longer ahead. expect(ahead(120)).toEqual([]); }); it('at the start are ahead too when it loops and its end comes close – unless still alive', () => { const cut = makeCut(); const screen = videoSprite(cut, addVideo(cut)); const track = firstTrack(cut); const first = cut.item(track, screen, 0, 120); cut.item(track, screen, 120, 120); const ids = (frame: number) => compositionAhead(cut.project, cut.comp(), frame).objects.map((o) => o.id); expect(ids(200)).toEqual([]); cut.comp().props[COMPOSITION_LOOP_KEY] = true; expect(ids(179)).toEqual([]); expect(ids(200)).toEqual([first]); cut.comp().items[0]!.length = 240; cut.comp().items.pop(); expect(ids(200)).toEqual([]); }); it('follow the texture an override gives the item', () => { const cut = makeCut(); const item = cut.item(firstTrack(cut), cut.spriteId, 0, 1); cut.comp().objects[0]!.overrides.texture = addVideo(cut, 4); expect( fittedLength( cut.project, cut.comp(), cut.comp().items.find((i) => i.id === item)!, ), ).toBe(240); }); }); describe('saving and loading a composition', () => { it('keeps tracks, items, clips and the objects', () => { const cut = makeCut(); const track = cut.track('Second'); cut.item(track, cut.spriteId, 12, 34, { offset: 1.25, fitMedia: true }); cut.clip(track, 'gone', 5); const loaded = findAssetOf(roundTrip(cut.project), cut.compId, 'composition')!; expect(loaded.tracks).toEqual(cut.comp().tracks); expect(loaded.items).toEqual(cut.comp().items); expect(loaded.clips).toEqual(cut.comp().clips); expect(loaded.objects).toEqual(cut.comp().objects); }); it('drops what no item spawns, an item without its object or track, and what is no time', () => { const cut = makeCut(); const track = firstTrack(cut); const kept = cut.item(track, cut.spriteId, 0, 10); cut.comp().objects.push(createSceneObject('stray', cut.spriteId, { x: 0, y: 0 })); cut.comp().items.push({ id: 'no-object', track, start: 0, length: 10, offset: 0 }); cut.item('no-track', cut.spriteId, 0, 10); cut.item(track, cut.spriteId, -5, 10); cut.item(track, cut.spriteId, 0, 0); cut.clip('no-track', 'x', 0); cut.clip(track, 'y', -1); const loaded = findAssetOf(roundTrip(cut.project), cut.compId, 'composition')!; expect(loaded.items.map((i) => i.id)).toEqual([kept]); expect(loaded.objects.map((o) => o.id)).toEqual([kept]); expect(loaded.clips).toEqual([]); }); it('keeps every object at the top level, a negative offset, and at least one track', () => { const cut = makeCut(); const track = firstTrack(cut); const parent = cut.item(track, cut.spriteId, 0, 10); const child = cut.item(track, cut.spriteId, 10, 10, { offset: -3 }); cut.comp().objects.find((o) => o.id === child)!.parentId = parent; const raw = structuredClone(cut.project); findAssetOf(raw, cut.compId, 'composition')!.tracks = []; const loaded = findAssetOf(roundTrip(raw), cut.compId, 'composition')!; expect(loaded.tracks).toHaveLength(1); // The one track left is a new one, so the items on the old one fall away. expect(loaded.items).toEqual([]); const again = findAssetOf(roundTrip(cut.project), cut.compId, 'composition')!; expect(again.objects.map((o) => o.parentId)).toEqual(['', '']); expect(again.items.find((i) => i.id === child)!.offset).toBe(-3); expect(again.items.some((i) => 'fitMedia' in i)).toBe(false); }); });