diff --git a/src/core/commands/assetCommands.ts b/src/core/commands/assetCommands.ts index 50caae29..bf23db23 100644 --- a/src/core/commands/assetCommands.ts +++ b/src/core/commands/assetCommands.ts @@ -16,6 +16,7 @@ import type { AssetType } from '../model/AssetTypes'; import { embeddedIn } from '../music/embedded'; import { defaultTracks } from '../music/song'; import { heldSongGraph } from '../music/routing'; +import { newCompositionTrack } from '../model/composition'; import { linearCurve } from '../model/curve'; import { isFontFile } from '../model/font'; import { DEFAULT_PALETTE, PIXEL_CANVAS_DEFAULT } from '../model/pixelart'; @@ -126,6 +127,15 @@ export function buildAsset(type: AssetType, name: string, id = newId()): Asset { return { ...base, type: 'bus' }; case 'particles': return { ...base, type: 'particles', graph: createEmptyGraph() }; + case 'composition': + return { + ...base, + type: 'composition', + objects: [], + tracks: [newCompositionTrack(1)], + items: [], + clips: [], + }; } } @@ -221,13 +231,15 @@ function copyOf(original: Asset, id: string, name: string): Asset { if (copy.type === 'texture' || copy.type === 'pixelart') for (const frame of copy.frames) frame.id = newId(); renewClipIds(copy, new Map()); - if (copy.type === 'scene' || copy.type === 'container') { + if (copy.type === 'scene' || copy.type === 'container' || copy.type === 'composition') { const ids = new Map(copy.objects.map((o) => [o.id, newId()])); for (const object of copy.objects) { object.id = ids.get(object.id)!; object.parentId = ids.get(object.parentId) ?? object.parentId; } swapReferences(copy.objects, (ref) => renamedReference(ref, ids), new Recorder()); + // An item spawns the object of its own id. + if (copy.type === 'composition') for (const item of copy.items) item.id = ids.get(item.id) ?? item.id; } return copy; } diff --git a/src/core/model/Asset.ts b/src/core/model/Asset.ts index 1ff654c1..4e98540c 100644 --- a/src/core/model/Asset.ts +++ b/src/core/model/Asset.ts @@ -260,6 +260,53 @@ export interface AnimationAsset extends AssetBase { tracks: AnimationTrack[]; } +/** A lane of a composition (#417); the first one lies in front. */ +export interface CompositionTrack { + id: string; + name: string; +} + +/** + * One stretch on a track: the object of the same id lives from `start` for `length` frames. + * `offset` is how many seconds into its video the item begins, which a trim of the left edge moves. + * `fitMedia` says its length still follows that video, or the default while there is none. + */ +export interface CompositionItem { + id: string; + track: string; + start: number; + length: number; + offset: number; + fitMedia?: boolean; +} + +/** + * An animation laid on a track from `start`, one pass long. It plays on whichever item of its track + * it overlaps; its rows find their targets there as `Play animation` finds them. + */ +export interface CompositionClip { + id: string; + track: string; + animation: string; + start: number; +} + +/** + * Objects along a time axis (#417), for a cut or a cutscene: each item spawns its object for as long + * as it lasts, and animation clips play on the items they overlap. The objects are stored like a + * scene's, overrides and container edits included, one per item and all at the top level. Time is + * counted in frames of its own `fps` (see `model/composition.ts`). + */ +export interface CompositionAsset extends AssetBase { + type: 'composition'; + objects: SceneObject[]; + /** Hierarchy folders, as a scene's (#241); absent when there are none. */ + groups?: ObjectGroup[]; + tracks: CompositionTrack[]; + items: CompositionItem[]; + clips: CompositionClip[]; +} + /** * The graph holds one `State` node per state and an edge per transition; a behavior reads the * machine's current state through `State branch` and moves it through `Set state` or `State transition`. @@ -670,6 +717,7 @@ export type Asset = | PixelArtAsset | PaletteAsset | AnimationAsset + | CompositionAsset | CurveAsset | TileAsset | TilemapAsset diff --git a/src/core/model/AssetTypes.ts b/src/core/model/AssetTypes.ts index f826bf61..fff75d02 100644 --- a/src/core/model/AssetTypes.ts +++ b/src/core/model/AssetTypes.ts @@ -31,6 +31,7 @@ export const ASSET_TYPES = [ 'bus', 'particles', 'video', + 'composition', ] as const; export type AssetType = (typeof ASSET_TYPES)[number]; @@ -68,6 +69,7 @@ export const ASSET_TYPE_LABELS: Record = { bus: 'Bus', particles: 'Particles', video: 'Video', + composition: 'Composition', }; /** @@ -82,7 +84,8 @@ export const ASSET_TYPE_LABELS: Record = { * sound is played through (#311), heard but never seen, an instrument what a note is played * with (#331), a pattern the notes it plays (#333) and a song what lays them out (#334). A bus is * where sounds are mixed on their way out (#325). A video is a moving picture a sprite wears through - * `texture` (#402), as it wears an image. + * `texture` (#402), as it wears an image. A composition is not placed yet: what a placed one shows + * and plays is still to come (#417). */ export type PlaceableAssetType = Exclude< AssetType, @@ -107,6 +110,7 @@ export type PlaceableAssetType = Exclude< | 'variables' | 'struct' | 'video' + | 'composition' >; const NOT_PLACEABLE: ReadonlySet = new Set([ @@ -131,6 +135,7 @@ const NOT_PLACEABLE: ReadonlySet = new Set([ 'variables', 'struct', 'video', + 'composition', ]); export function isPlaceableType(type: AssetType): type is PlaceableAssetType { diff --git a/src/core/model/Project.ts b/src/core/model/Project.ts index 9ff706f8..813db86d 100644 --- a/src/core/model/Project.ts +++ b/src/core/model/Project.ts @@ -1,5 +1,6 @@ import { type Asset, + type CompositionAsset, type ContainerAsset, type GraphAsset, isGraphAsset, @@ -151,13 +152,14 @@ export function findGraphAsset(project: Project, assetId: string): GraphHolder | } /** - * Everything that holds placed objects. A container's `objects` are a template, so anything - * walking "all placed objects" must cover both or it silently skips the templates. + * Everything that holds placed objects. A container's `objects` are a template, and a composition's + * live only while their items run, so anything walking "all placed objects" must cover all three or + * it silently skips some. */ -export type ObjectHolder = SceneAsset | ContainerAsset; +export type ObjectHolder = SceneAsset | ContainerAsset | CompositionAsset; function isObjectHolder(asset: Asset): asset is ObjectHolder { - return asset.type === 'scene' || asset.type === 'container'; + return asset.type === 'scene' || asset.type === 'container' || asset.type === 'composition'; } export function objectHolders(project: Project): ObjectHolder[] { diff --git a/src/core/model/animationPreview.ts b/src/core/model/animationPreview.ts index 1e5448ba..f7356c3e 100644 --- a/src/core/model/animationPreview.ts +++ b/src/core/model/animationPreview.ts @@ -41,7 +41,7 @@ export function previewHolder( const at: ReadAt = { frame, reverse, curves: curveLookup(project) }; for (const track of reset ? [] : animation.tracks) { if (root.type === 'scene' && track.target === '') writeScene(holder, track, at); - else writeObject(project, holder, previewObjectId(root, track.target), track, at); + else writeObjectRow(project, holder, previewObjectId(root, track.target), track, at); } return holder; } @@ -71,7 +71,7 @@ function placedHolder(root: Asset): ObjectHolder { } /** Where on the timeline a row is read, and the Curve assets its keys may blend along. */ -interface ReadAt { +export interface ReadAt { frame: number; reverse: boolean; curves: CurveLookup; @@ -86,7 +86,8 @@ function writeScene(holder: ObjectHolder, track: AnimationTrack, at: ReadAt): vo if (value !== undefined) holder.props[track.property] = value; } -function writeObject( +/** A row on the object `id` of the holder, stored there or resolved from a container – also a composition's (#417). */ +export function writeObjectRow( project: Project, holder: ObjectHolder, id: string, diff --git a/src/core/model/composition.ts b/src/core/model/composition.ts new file mode 100644 index 00000000..dfb1f161 --- /dev/null +++ b/src/core/model/composition.ts @@ -0,0 +1,303 @@ +/** + * Compositions (#417): objects along a time axis, counted in frames of the composition's own `fps`. + * What runs at a frame is one pure function, `compositionAt`, so the tab's preview and the running + * game read the same cut; `compositionHolder` writes it out as objects for a view to draw. + */ +import { newId } from '../ids'; +import { + COMPOSITION_FPS_KEY, + COMPOSITION_LOOP_KEY, + DEFAULT_COMPOSITION_FPS, +} from '../properties/compositions'; +import { resolveSceneObjectProps } from '../properties/resolve'; +import { VIDEO_CLOCK_KEY, VIDEO_TIME_KEY, videoDrivenOf } from '../properties/videos'; +import { animationDuration, fpsOf, frameCount, targetNamePath } from './animation'; +import { type ReadAt, writeObjectRow } from './animationPreview'; +import type { + AnimationAsset, + AnimationTrack, + CompositionAsset, + CompositionClip, + CompositionItem, + CompositionTrack, +} from './Asset'; +import { curveLookup } from './curve'; +import { instanceChildId, objectRef, resolveObjects, splitInstanceId } from './instance'; +import { findAsset, findAssetOf, type Project } from './Project'; +import { booleanOf, type PropertyBag } from './Properties'; +import { objectName, type SceneObject } from './SceneObject'; +import { findVideo, videoLengthOf, type VideoRef } from './video'; + +/** How long an item lasts while it has no video to take its length from. */ +export const DEFAULT_ITEM_SECONDS = 3; + +export function compositionFps(comp: CompositionAsset): number { + const fps = Number(comp.props[COMPOSITION_FPS_KEY]); + return Number.isFinite(fps) && fps > 0 ? fps : DEFAULT_COMPOSITION_FPS; +} + +export function compositionLoops(comp: CompositionAsset): boolean { + return booleanOf(comp.props[COMPOSITION_LOOP_KEY]); +} + +export function newCompositionTrack(index: number): CompositionTrack { + return { id: newId(), name: `Track ${index}` }; +} + +export function itemEnd(item: CompositionItem): number { + return item.start + item.length; +} + +/** The object an item spawns; undefined once it is gone, and then the item does nothing. */ +export function itemObject(comp: CompositionAsset, item: CompositionItem): SceneObject | undefined { + return comp.objects.find((o) => o.id === item.id); +} + +/** + * The animation a clip plays, unless it is gone or built over a scene: a scene's rows aim at that + * scene's objects, nothing an item holds. + */ +function clipAnimation(project: Project, clip: CompositionClip): AnimationAsset | undefined { + const animation = findAssetOf(project, clip.animation, 'animation'); + return animation && findAsset(project, animation.root)?.type !== 'scene' ? animation : undefined; +} + +/** Frames one pass of the clip's animation takes in the composition; 0 when it plays nothing. */ +export function clipLength(project: Project, comp: CompositionAsset, clip: CompositionClip): number { + const animation = clipAnimation(project, clip); + const seconds = animation ? animationDuration(animation) : 0; + return seconds > 0 ? Math.max(1, Math.ceil(seconds * compositionFps(comp) - 1e-6)) : 0; +} + +/** The frame after the last item or clip ends: how long one run of the composition is. */ +export function compositionLength(project: Project, comp: CompositionAsset): number { + let end = 0; + for (const item of comp.items) if (itemObject(comp, item)) end = Math.max(end, itemEnd(item)); + for (const clip of comp.clips) end = Math.max(end, clip.start + clipLength(project, comp, clip)); + return end; +} + +/** + * The video the item's own object wears through its texture – the override's, else its sprite's. A + * video worn further down a container item runs with the item but gives it no length. + */ +export function itemVideo(project: Project, comp: CompositionAsset, item: CompositionItem): VideoRef | null { + const object = itemObject(comp, item); + const asset = object && findAsset(project, object.assetId); + if (!object || asset?.type !== 'sprite') return null; + return videoWorn(resolveSceneObjectProps(project, asset, object), project); +} + +function videoWorn(props: PropertyBag, project: Project): VideoRef | null { + const texture = props.texture; + return typeof texture === 'string' ? findVideo(project, texture) : null; +} + +/** + * The length an item with `fitMedia` takes: what is left of its video after `offset`, or the default + * while it wears none or the video's length is not known yet. + */ +export function fittedLength(project: Project, comp: CompositionAsset, item: CompositionItem): number { + const fps = compositionFps(comp); + const video = itemVideo(project, comp, item); + const seconds = video ? videoLengthOf(video) - item.offset : 0; + return seconds > 0 ? Math.max(1, Math.round(seconds * fps)) : Math.round(DEFAULT_ITEM_SECONDS * fps); +} + +/** A clip playing on an item, at `frame` of its animation – held on the last one once it is through. */ +export interface ClipAt { + clip: CompositionClip; + animation: AnimationAsset; + frame: number; +} + +/** An item alive at a frame of the composition. */ +export interface LiveItem { + item: CompositionItem; + object: SceneObject; + /** The index of its track, 0 the first. */ + track: number; + /** Seconds since the item began. */ + time: number; + /** Seconds into what its videos play: `time` past the item's offset. */ + videoTime: number; + /** In the order they started; a later one writes over an earlier one. */ + clips: ClipAt[]; +} + +/** + * Everything alive at `frame`, in drawing order: the last track first, so the first lies in front – + * the order the items take below the composition, which decides between equal `z`. An item lives + * from its start up to, not including, its end; one on a track that is gone, or whose object is gone, + * does not. A clip plays on an item of its track it overlaps once it has started, and holds its last + * frame for as long as that item lasts. + */ +export function compositionAt(project: Project, comp: CompositionAsset, frame: number): LiveItem[] { + const fps = compositionFps(comp); + const tracks = new Map(comp.tracks.map((t, index) => [t.id, index])); + const clips = comp.clips + .map((clip, order) => ({ clip, order, animation: clipAnimation(project, clip) })) + .filter((c) => c.animation !== undefined && c.clip.start <= frame) + .sort((a, b) => a.clip.start - b.clip.start || a.order - b.order); + const live: { item: LiveItem; order: number }[] = []; + comp.items.forEach((item, order) => { + const track = tracks.get(item.track); + const object = itemObject(comp, item); + if (track === undefined || !object || frame < item.start || frame >= itemEnd(item)) return; + const time = (frame - item.start) / fps; + const playing: ClipAt[] = []; + for (const { clip, animation } of clips) { + const end = clip.start + clipLength(project, comp, clip); + if (clip.track !== item.track || clip.start >= itemEnd(item) || end <= item.start) continue; + const at = ((frame - clip.start) / fps) * fpsOf(animation!); + playing.push({ clip, animation: animation!, frame: Math.min(at, frameCount(animation!) - 1) }); + } + live.push({ item: { item, object, track, time, videoTime: item.offset + time, clips: playing }, order }); + }); + live.sort( + (a, b) => b.item.track - a.item.track || a.item.item.start - b.item.item.start || a.order - b.order, + ); + return live.map((l) => l.item); +} + +/** + * The composition at `frame` as objects a view draws: a copy holding only what is alive, in drawing + * order, each with its clips' rows and its video time written over it. Every sprite of an item that + * wears a video and is not driven by hand is driven at the item's video time. Nothing here touches + * the document. + */ +export function compositionHolder(project: Project, comp: CompositionAsset, frame: number): CompositionAsset { + const live = compositionAt(project, comp, frame); + const holder: CompositionAsset = { + ...comp, + props: { ...comp.props }, + objects: live.map((l) => copyOf(l.object)), + }; + const curves = curveLookup(project); + for (const item of live) { + driveVideos(project, holder, item); + for (const clip of item.clips) { + const at: ReadAt = { frame: clip.frame, reverse: false, curves }; + for (const track of clip.animation.tracks) { + const id = rowTarget(project, holder, item.object.id, clip.animation, track); + if (id) writeObjectRow(project, holder, id, track, at); + } + } + } + return holder; +} + +const record = (value: unknown): Record => + typeof value === 'object' && value !== null ? (value as Record) : {}; +const text = (value: unknown) => (typeof value === 'string' ? value : ''); +const finite = (value: unknown) => (typeof value === 'number' && Number.isFinite(value) ? value : NaN); +const list = (raw: unknown): Record[] => (Array.isArray(raw) ? raw.map(record) : []); + +/** Held to what a track may be: ids no other has, a name, and at least one track. */ +export function heldCompositionTracks(raw: unknown): CompositionTrack[] { + const tracks: CompositionTrack[] = []; + for (const t of list(raw)) { + const id = text(t.id); + if (!id || tracks.some((other) => other.id === id)) continue; + tracks.push({ id, name: text(t.name) || `Track ${tracks.length + 1}` }); + } + return tracks.length > 0 ? tracks : [newCompositionTrack(1)]; +} + +/** + * Held to what an item may be: the id of an object it spawns and no other item does, a track of + * the composition, a start in whole frames from 0, at least a frame long, an offset from 0. + */ +export function heldCompositionItems( + raw: unknown, + tracks: readonly CompositionTrack[], + objects: readonly SceneObject[], +): CompositionItem[] { + const known = new Set(tracks.map((t) => t.id)); + const placed = new Set(objects.map((o) => o.id)); + const items: CompositionItem[] = []; + for (const i of list(raw)) { + const id = text(i.id); + const start = Math.round(finite(i.start)); + const length = Math.round(finite(i.length)); + if (!placed.has(id) || items.some((other) => other.id === id) || !known.has(text(i.track))) continue; + if (!(start >= 0) || !(length >= 1)) continue; + const offset = finite(i.offset); + const item: CompositionItem = { + id, + track: text(i.track), + start, + length, + offset: offset > 0 ? offset : 0, + }; + if (i.fitMedia === true) item.fitMedia = true; + items.push(item); + } + return items; +} + +/** Held like an item; an animation that is gone is kept, as an animation keeps a root that is gone. */ +export function heldCompositionClips(raw: unknown, tracks: readonly CompositionTrack[]): CompositionClip[] { + const known = new Set(tracks.map((t) => t.id)); + const clips: CompositionClip[] = []; + for (const c of list(raw)) { + const id = text(c.id); + const start = Math.round(finite(c.start)); + if (!id || clips.some((other) => other.id === id) || !known.has(text(c.track)) || !(start >= 0)) continue; + clips.push({ id, track: text(c.track), animation: text(c.animation), start }); + } + return clips; +} + +function copyOf(object: SceneObject): SceneObject { + return { + ...object, + position: { ...object.position }, + overrides: { ...object.overrides }, + instanceEdits: structuredClone(object.instanceEdits), + }; +} + +function driveVideos(project: Project, holder: CompositionAsset, live: LiveItem): void { + const below = resolveObjects(project, holder).filter( + (o) => splitInstanceId(o.id)?.instanceId === live.object.id, + ); + for (const object of [holder.objects.find((o) => o.id === live.object.id)!, ...below]) { + const asset = findAsset(project, object.assetId); + if (asset?.type !== 'sprite') continue; + const props = resolveSceneObjectProps(project, asset, object); + const own = object.id === live.object.id; + if (!videoWorn(props, project) || (!own && videoDrivenOf(props))) continue; + const ref = objectRef(project, holder, object.id); + ref?.setOverride(VIDEO_CLOCK_KEY, 'driven'); + ref?.setOverride(VIDEO_TIME_KEY, live.videoTime); + } +} + +/** + * The object of the holder a row of a clip aims at on the item `rootId`: the item itself, the object + * at the row's template path below it, or else the one its names lead to – the way `Play animation` + * finds a target on an object built from another container. + */ +function rowTarget( + project: Project, + holder: CompositionAsset, + rootId: string, + animation: AnimationAsset, + track: AnimationTrack, +): string | undefined { + if (track.target === '') return rootId; + const objects = resolveObjects(project, holder); + const byPath = instanceChildId(rootId, track.target); + if (objects.some((o) => o.id === byPath)) return byPath; + const names = targetNamePath(project, findAsset(project, animation.root), track.target); + let at: string | undefined = rootId; + for (const name of names ?? []) { + const found = objects.filter( + (o) => o.parentId === at && objectName(o, findAsset(project, o.assetId)) === name, + ); + at = found.length === 1 ? found[0]!.id : undefined; + if (!at) return undefined; + } + return names ? at : undefined; +} diff --git a/src/core/properties/PropertySchema.ts b/src/core/properties/PropertySchema.ts index 2f070e57..8d7dc705 100644 --- a/src/core/properties/PropertySchema.ts +++ b/src/core/properties/PropertySchema.ts @@ -23,6 +23,7 @@ import { } from './kinds'; import { findAssetOf, type Project } from '../model/Project'; import { ALL_LAYERS, COLLIDES_WITH_KEY, COLLISION_LAYER_KEY } from '../model/collisionLayers'; +import { COMPOSITION_FPS_KEY, COMPOSITION_LOOP_KEY, DEFAULT_COMPOSITION_FPS } from './compositions'; import { INHERIT_LAYER, OVERLAY_KEY, RENDER_LAYER_KEY, SEES_KEY } from '../model/renderLayers'; import { TILE_BOUNCINESS_KEY, @@ -597,6 +598,21 @@ const ASSET_SCHEMAS: Record = { ], }, ], + // Tracks, items and clips are intrinsic data on the asset (see CompositionAsset). Whole frames + // per second, so a cut lands on a frame. + composition: [ + { + key: COMPOSITION_FPS_KEY, + label: 'Frames per second', + kind: 'number', + default: DEFAULT_COMPOSITION_FPS, + min: 1, + max: 120, + step: 1, + decimals: 0, + }, + { key: COMPOSITION_LOOP_KEY, label: 'Loop', kind: 'boolean', default: false }, + ], // The pictures are intrinsic data on the asset (see TileAsset); these three say what the // block holds and how its steps run. There is no tile size: the block's width in tiles is known // from the kind, so the picture already says it (see `readTile`). A new tile is a plain one; the diff --git a/src/core/properties/compositions.ts b/src/core/properties/compositions.ts new file mode 100644 index 00000000..6b63757f --- /dev/null +++ b/src/core/properties/compositions.ts @@ -0,0 +1,4 @@ +/** The rows of a composition (#417), apart from its model so the schema can read them without a cycle. */ +export const COMPOSITION_FPS_KEY = 'fps'; +export const COMPOSITION_LOOP_KEY = 'loop'; +export const DEFAULT_COMPOSITION_FPS = 60; diff --git a/src/core/store/serialize.ts b/src/core/store/serialize.ts index f625756a..1cf4b891 100644 --- a/src/core/store/serialize.ts +++ b/src/core/store/serialize.ts @@ -36,6 +36,7 @@ import type { AnimationTrack, Asset, BehaviorAsset, + CompositionAsset, ContainerAsset, PictureCut, PixelArtAsset, @@ -72,7 +73,8 @@ import { isSoundFile } from '../model/sound'; import { isVideoFile } from '../model/video'; import { emptyEdit, type InstanceEdit, isEmptyEdit } from '../model/instance'; import { type PropertyBag, type PropertyValue, tagOf, type Vec2 } from '../model/Properties'; -import { PROJECT_FORMAT_VERSION, type Project } from '../model/Project'; +import { type ObjectHolder, PROJECT_FORMAT_VERSION, type Project } from '../model/Project'; +import { heldCompositionClips, heldCompositionItems, heldCompositionTracks } from '../model/composition'; import { readScope, readSharedScope } from '../model/Scope'; import type { ObjectGroup, SceneObject } from '../model/SceneObject'; import { @@ -205,6 +207,24 @@ export function deserializeProject(raw: unknown, fallbackId = ''): Project { }; normalizeGroups(holder, a.groups); assets.push(holder); + } else if (type === 'composition') { + // One object per item, all at the top level; an object no item spawns is dropped with it. + const objects = normalizeObjects(a.objects, schemaOf).map((o) => ({ ...o, parentId: '' })); + const tracks = heldCompositionTracks(a.tracks); + const items = heldCompositionItems(a.items, tracks, objects); + const spawned = new Set(items.map((i) => i.id)); + const composition: CompositionAsset = { + id, + type, + name, + props, + objects: objects.filter((o) => spawned.has(o.id)), + tracks, + items, + clips: heldCompositionClips(a.clips, tracks), + }; + normalizeGroups(composition, a.groups); + assets.push(composition); } else if (type === 'behavior') { const declared = variables.get(id) ?? []; const behavior: BehaviorAsset = { @@ -447,7 +467,7 @@ function normalizeObjects( * Hierarchy folders (#241): only those with a member are kept, and a member is a top-level object * naming one of them – anything else loses its `groupId`. */ -function normalizeGroups(holder: SceneAsset | ContainerAsset, raw: unknown): void { +function normalizeGroups(holder: ObjectHolder, raw: unknown): void { const groups = new Map(); for (const item of Array.isArray(raw) ? raw : []) { const g = asRecord(item); diff --git a/src/editor/panels/assetGroups.ts b/src/editor/panels/assetGroups.ts index b3044f00..c7fff349 100644 --- a/src/editor/panels/assetGroups.ts +++ b/src/editor/panels/assetGroups.ts @@ -23,6 +23,7 @@ export const ASSET_GROUPS: Record = { camera, clip, collider, + composition, container, curve, font, diff --git a/src/editor/panels/icons/composition.svg b/src/editor/panels/icons/composition.svg new file mode 100644 index 00000000..e0bd261e --- /dev/null +++ b/src/editor/panels/icons/composition.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/tests/core/composition.test.ts b/tests/core/composition.test.ts new file mode 100644 index 00000000..fcfae15a --- /dev/null +++ b/tests/core/composition.test.ts @@ -0,0 +1,596 @@ +/** + * Compositions (#417): objects along a time axis. What lives at a frame, which clips play on it and + * where its videos stand is `compositionAt`; `compositionHolder` writes it out for a view to draw. + */ +import { describe, expect, it } from 'vitest'; +import { + AddAnimationTrackCommand, + SetAnimationKeyCommand, + SetAnimationRootCommand, + SetTrackRelativeCommand, +} from '../../src/core/commands/animationCommands'; +import { + CreateAssetCommand, + CreateVideoCommand, + DeleteAssetCommand, + DuplicateAssetCommand, +} from '../../src/core/commands/assetCommands'; +import { AddClipCommand } from '../../src/core/commands/clipCommands'; +import type { + AnimationAsset, + CompositionAsset, + CompositionItem, + ContainerAsset, +} from '../../src/core/model/Asset'; +import { + clipLength, + compositionAt, + compositionFps, + compositionHolder, + compositionLength, + compositionLoops, + fittedLength, + itemVideo, +} from '../../src/core/model/composition'; +import { instanceChildId, resolveObjects } from '../../src/core/model/instance'; +import { findAssetOf, objectHolders } from '../../src/core/model/Project'; +import type { PropertyBag } from '../../src/core/model/Properties'; +import { createSceneObject } from '../../src/core/model/SceneObject'; +import { + VIDEO_CLOCK_KEY, + VIDEO_END_KEY, + VIDEO_START_KEY, + VIDEO_TIME_KEY, +} from '../../src/core/properties/videos'; +import { makeProject, type ProjectFixture, roundTrip } from '../helpers'; + +const FILE = `a1b2c3d4e5f6/${'c7'.repeat(32)}.mp4`; + +interface Cut extends ProjectFixture { + compId: string; + spriteId: string; + comp(): CompositionAsset; + /** Adds a track and returns its id. */ + track(name: string): string; + /** Places `assetId` as an item on a track; returns the item's id, which is its object's. */ + item( + track: string, + assetId: string, + start: number, + length: number, + extra?: Partial, + ): string; + animation( + root: string, + rows: [property: string, keys: [number, number][], target?: string][], + fps?: number, + ): string; + clip(track: string, animation: string, start: number): string; +} + +function makeCut(): Cut { + const f = makeProject(); + const comp = new CreateAssetCommand('composition', 'Cut'); + const sprite = new CreateAssetCommand('sprite', 'Box'); + f.stack.execute(comp); + f.stack.execute(sprite); + const current = () => findAssetOf(f.project, comp.asset.id, 'composition')!; + let next = 0; + return { + ...f, + compId: comp.asset.id, + spriteId: sprite.asset.id, + comp: current, + track(name) { + const id = `t${++next}`; + current().tracks.push({ id, name }); + return id; + }, + item(track, assetId, start, length, extra = {}) { + const id = `i${++next}`; + current().objects.push(createSceneObject(id, assetId, { x: 0, y: 0 })); + current().items.push({ id, track, start, length, offset: 0, ...extra }); + return id; + }, + animation(root, rows, fps = 10) { + const create = new CreateAssetCommand('animation', 'Move', { fps, playback: 'loop' }); + f.stack.execute(create); + f.stack.execute(new SetAnimationRootCommand(create.asset.id, root)); + for (const [property, keys, target = ''] of rows) { + const row = new AddAnimationTrackCommand(create.asset.id, target, property); + f.stack.execute(row); + for (const [frame, value] of keys) + f.stack.execute(new SetAnimationKeyCommand(create.asset.id, row.trackId, frame, value)); + } + return create.asset.id; + }, + clip(track, animation, start) { + const id = `c${++next}`; + current().clips.push({ id, track, animation, start }); + return id; + }, + }; +} + +const firstTrack = (cut: Cut) => cut.comp().tracks[0]!.id; +const liveIds = (cut: Cut, frame: number) => + compositionAt(cut.project, cut.comp(), frame).map((l) => l.object.id); + +function addVideo(cut: Cut, duration = 10): string { + const video = new CreateVideoCommand('Intro', FILE, { + width: 320, + height: 180, + duration, + fps: 30, + audio: false, + }); + cut.stack.execute(video); + return video.asset.id; +} + +function videoSprite(cut: Cut, texture: string): string { + const sprite = new CreateAssetCommand('sprite', 'Screen', { texture }); + cut.stack.execute(sprite); + return sprite.asset.id; +} + +/** A container "Rig" holding a sprite "Arm", returned with the template id of the arm. */ +function rig(cut: Cut, name = 'Rig', armProps: PropertyBag = {}): { id: string; arm: string } { + const create = new CreateAssetCommand('container', name); + cut.stack.execute(create); + const arm = new CreateAssetCommand('sprite', 'Arm', armProps); + cut.stack.execute(arm); + const template = findAssetOf(cut.project, create.asset.id, 'container') as ContainerAsset; + const armId = `arm-of-${name}`; + template.objects.push(createSceneObject(armId, arm.asset.id, { x: 5, y: 0 })); + return { id: create.asset.id, arm: armId }; +} + +describe('a new composition', () => { + it('starts with one track, nothing on it, at 60 fps and not looping', () => { + const cut = makeCut(); + const comp = cut.comp(); + expect(comp.tracks).toHaveLength(1); + expect(comp.tracks[0]!.name).toBe('Track 1'); + expect([comp.objects, comp.items, comp.clips]).toEqual([[], [], []]); + expect(compositionFps(comp)).toBe(60); + expect(compositionLoops(comp)).toBe(false); + }); + + it('is an object holder, so deleting an asset takes the object it placed – and undo brings it back', () => { + const cut = makeCut(); + const item = cut.item(firstTrack(cut), cut.spriteId, 0, 60); + expect(objectHolders(cut.project).map((h) => h.id)).toContain(cut.compId); + cut.stack.execute(new DeleteAssetCommand(cut.spriteId)); + expect(cut.comp().objects).toEqual([]); + expect(liveIds(cut, 10)).toEqual([]); + cut.stack.undo(); + expect(liveIds(cut, 10)).toEqual([item]); + }); + + it('a duplicate gets objects of its own, and its items follow them', () => { + const cut = makeCut(); + cut.item(firstTrack(cut), cut.spriteId, 0, 60); + const duplicate = new DuplicateAssetCommand(cut.compId); + cut.stack.execute(duplicate); + const copy = findAssetOf(cut.project, duplicate.copyId, 'composition')!; + expect(copy.objects[0]!.id).not.toBe(cut.comp().objects[0]!.id); + expect(copy.items[0]!.id).toBe(copy.objects[0]!.id); + expect(compositionAt(cut.project, copy, 10).map((l) => l.object.id)).toEqual([copy.objects[0]!.id]); + }); +}); + +describe('what lives at a frame', () => { + it('an item from its start up to, not including, its end', () => { + const cut = makeCut(); + const item = cut.item(firstTrack(cut), cut.spriteId, 30, 60); + expect(liveIds(cut, 29)).toEqual([]); + expect(liveIds(cut, 30)).toEqual([item]); + expect(liveIds(cut, 89)).toEqual([item]); + expect(liveIds(cut, 90)).toEqual([]); + }); + + it('counts its time in seconds since its start, and its video time from its offset', () => { + const cut = makeCut(); + cut.item(firstTrack(cut), cut.spriteId, 30, 120, { offset: 2 }); + const [live] = compositionAt(cut.project, cut.comp(), 60); + expect(live!.time).toBe(0.5); + expect(live!.videoTime).toBe(2.5); + }); + + it('nothing of an item whose object or track is gone', () => { + const cut = makeCut(); + const kept = cut.item(firstTrack(cut), cut.spriteId, 0, 60); + const orphan = cut.item(firstTrack(cut), cut.spriteId, 0, 60); + cut.comp().objects = cut.comp().objects.filter((o) => o.id !== orphan); + cut.item('no-such-track', cut.spriteId, 0, 60); + expect(liveIds(cut, 10)).toEqual([kept]); + }); + + it('draws the last track first, so the first lies in front; a track by start', () => { + const cut = makeCut(); + const top = firstTrack(cut); + const bottom = cut.track('Background'); + const front = cut.item(top, cut.spriteId, 0, 60); + const backLate = cut.item(bottom, cut.spriteId, 10, 60); + const backEarly = cut.item(bottom, cut.spriteId, 0, 30); + expect(liveIds(cut, 20)).toEqual([backEarly, backLate, front]); + expect( + resolveObjects(cut.project, compositionHolder(cut.project, cut.comp(), 20)).map((o) => o.id), + ).toEqual([backEarly, backLate, front]); + }); + + it('lasts until the last item or clip ends', () => { + const cut = makeCut(); + cut.item(firstTrack(cut), cut.spriteId, 0, 60); + expect(compositionLength(cut.project, cut.comp())).toBe(60); + cut.clip( + firstTrack(cut), + cut.animation(cut.spriteId, [ + [ + 'x', + [ + [0, 0], + [9, 90], + ], + ], + ]), + 40, + ); + expect(compositionLength(cut.project, cut.comp())).toBe(100); + }); +}); + +describe('animation clips', () => { + it('last one pass of their animation, counted in the frames of the composition', () => { + const cut = makeCut(); + // Keys on 0…9 at 10 fps: ten frames, one second – 60 frames at 60 fps. + const move = cut.animation(cut.spriteId, [ + [ + 'x', + [ + [0, 0], + [9, 90], + ], + ], + ]); + const clip = cut.clip(firstTrack(cut), move, 0); + expect( + clipLength( + cut.project, + cut.comp(), + cut.comp().clips.find((c) => c.id === clip)!, + ), + ).toBe(60); + }); + + it('play on the item of their track they overlap, from where the clip stands', () => { + const cut = makeCut(); + const item = cut.item(firstTrack(cut), cut.spriteId, 0, 120); + cut.clip( + firstTrack(cut), + cut.animation(cut.spriteId, [ + [ + 'x', + [ + [0, 0], + [9, 90], + ], + ], + ]), + 30, + ); + const x = (frame: number) => + compositionHolder(cut.project, cut.comp(), frame).objects.find((o) => o.id === item)!.position.x; + expect(x(29)).toBe(0); + expect(x(30)).toBe(0); + // 30 frames at 60 fps after its start: 0.5 s, frame 5 of the animation. + expect(x(60)).toBe(50); + expect(compositionAt(cut.project, cut.comp(), 60)[0]!.clips[0]!.frame).toBe(5); + }); + + it('hold their last frame once through, for as long as the item lasts', () => { + const cut = makeCut(); + const item = cut.item(firstTrack(cut), cut.spriteId, 0, 300); + cut.clip( + firstTrack(cut), + cut.animation(cut.spriteId, [ + [ + 'x', + [ + [0, 0], + [9, 90], + ], + ], + ]), + 0, + ); + const holder = compositionHolder(cut.project, cut.comp(), 200); + expect(holder.objects.find((o) => o.id === item)!.position.x).toBe(90); + }); + + it('play on neither an item of another track nor one that starts after they ended', () => { + const cut = makeCut(); + const other = cut.track('Other'); + const elsewhere = cut.item(other, cut.spriteId, 0, 300); + const later = cut.item(firstTrack(cut), cut.spriteId, 100, 60); + cut.clip( + firstTrack(cut), + cut.animation(cut.spriteId, [ + [ + 'x', + [ + [0, 10], + [9, 90], + ], + ], + ]), + 0, + ); + const at = compositionAt(cut.project, cut.comp(), 120); + expect(at.find((l) => l.object.id === elsewhere)!.clips).toEqual([]); + expect(at.find((l) => l.object.id === later)!.clips).toEqual([]); + }); + + it('running over two items, play on each in turn', () => { + const cut = makeCut(); + const track = firstTrack(cut); + const first = cut.item(track, cut.spriteId, 0, 30); + const second = cut.item(track, cut.spriteId, 30, 60); + cut.clip( + track, + cut.animation(cut.spriteId, [ + [ + 'x', + [ + [0, 0], + [9, 90], + ], + ], + ]), + 0, + ); + const x = (frame: number, id: string) => + compositionHolder(cut.project, cut.comp(), frame).objects.find((o) => o.id === id)!.position.x; + expect(x(12, first)).toBe(20); + expect(x(42, second)).toBe(70); + }); + + it('add a relative row to the value the item has', () => { + const cut = makeCut(); + const item = cut.item(firstTrack(cut), cut.spriteId, 0, 120); + cut.comp().objects[0]!.position.x = 100; + const move = cut.animation(cut.spriteId, [ + [ + 'x', + [ + [0, 0], + [9, 90], + ], + ], + ]); + const animation = findAssetOf(cut.project, move, 'animation') as AnimationAsset; + cut.stack.execute(new SetTrackRelativeCommand(move, animation.tracks[0]!.id, true)); + cut.clip(firstTrack(cut), move, 0); + const holder = compositionHolder(cut.project, cut.comp(), 30); + expect(holder.objects.find((o) => o.id === item)!.position.x).toBe(150); + }); + + it('reach into a container item by template path, and by names when built over another', () => { + const cut = makeCut(); + const own = rig(cut); + const lookalike = rig(cut, 'Other rig'); + const item = cut.item(firstTrack(cut), lookalike.id, 0, 120); + // Built over "Rig": its row names Rig's arm, which "Other rig" has by name only. + cut.clip( + firstTrack(cut), + cut.animation(own.id, [ + [ + 'rotation', + [ + [0, 0], + [9, 90], + ], + own.arm, + ], + ]), + 0, + ); + const holder = compositionHolder(cut.project, cut.comp(), 30); + const arm = resolveObjects(cut.project, holder).find( + (o) => o.id === instanceChildId(item, lookalike.arm), + )!; + expect(arm.overrides.rotation).toBe(50); + + const direct = makeCut(); + const same = rig(direct); + const placed = direct.item(firstTrack(direct), same.id, 0, 120); + direct.clip( + firstTrack(direct), + direct.animation(same.id, [ + [ + 'rotation', + [ + [0, 0], + [9, 90], + ], + same.arm, + ], + ]), + 0, + ); + const drawn = resolveObjects(direct.project, compositionHolder(direct.project, direct.comp(), 30)); + expect(drawn.find((o) => o.id === instanceChildId(placed, same.arm))!.overrides.rotation).toBe(50); + }); + + it('of an animation over a scene play nothing, and one gone neither', () => { + const cut = makeCut(); + const scene = new CreateAssetCommand('scene', 'Level'); + cut.stack.execute(scene); + cut.item(firstTrack(cut), cut.spriteId, 0, 120); + const overScene = cut.animation(scene.asset.id, [ + [ + 'gravityY', + [ + [0, 0], + [9, 90], + ], + ], + ]); + cut.clip(firstTrack(cut), overScene, 0); + cut.clip(firstTrack(cut), 'gone', 0); + expect(cut.comp().clips.map((c) => clipLength(cut.project, cut.comp(), c))).toEqual([0, 0]); + expect(compositionAt(cut.project, cut.comp(), 10)[0]!.clips).toEqual([]); + }); + + it('leave the document alone', () => { + const cut = makeCut(); + cut.item(firstTrack(cut), cut.spriteId, 0, 120); + cut.clip( + firstTrack(cut), + cut.animation(cut.spriteId, [ + [ + 'x', + [ + [0, 0], + [9, 90], + ], + ], + ]), + 0, + ); + const before = structuredClone(cut.comp()); + compositionHolder(cut.project, cut.comp(), 30); + expect(cut.comp()).toEqual(before); + }); +}); + +describe('items wearing a video', () => { + it('drive it at the item’s video time', () => { + const cut = makeCut(); + const screen = videoSprite(cut, addVideo(cut)); + const item = cut.item(firstTrack(cut), screen, 60, 600, { offset: 1.5 }); + const object = compositionHolder(cut.project, cut.comp(), 90).objects.find((o) => o.id === item)!; + expect(object.overrides[VIDEO_CLOCK_KEY]).toBe('driven'); + expect(object.overrides[VIDEO_TIME_KEY]).toBe(2); + }); + + it('drive one worn inside a container item too – unless that one is driven by hand', () => { + const cut = makeCut(); + const video = addVideo(cut); + const free = rig(cut, 'Free', { texture: video }); + const byHand = rig(cut, 'Hand', { texture: video, [VIDEO_CLOCK_KEY]: 'driven', [VIDEO_TIME_KEY]: 7 }); + const a = cut.item(firstTrack(cut), free.id, 0, 120); + const b = cut.item(cut.track('B'), byHand.id, 0, 120); + const drawn = resolveObjects(cut.project, compositionHolder(cut.project, cut.comp(), 60)); + const arm = (item: string, template: string) => + drawn.find((o) => o.id === instanceChildId(item, template))!; + expect(arm(a, free.arm).overrides[VIDEO_TIME_KEY]).toBe(1); + expect(arm(a, free.arm).overrides[VIDEO_CLOCK_KEY]).toBe('driven'); + // Its own 7 s stay: nothing is written over the sprite's driven time. + expect(arm(b, byHand.arm).overrides).toEqual({}); + }); + + it('take the length of the video, or of the clip, less the offset when fitted', () => { + const cut = makeCut(); + const video = addVideo(cut, 10); + const clip = new AddClipCommand(video, { [VIDEO_START_KEY]: 0.5, [VIDEO_END_KEY]: 2 }); + cut.stack.execute(clip); + const track = firstTrack(cut); + const whole = cut.item(track, videoSprite(cut, video), 0, 1, { fitMedia: true }); + const trimmed = cut.item(track, videoSprite(cut, video), 0, 1, { offset: 2 }); + const short = cut.item(track, videoSprite(cut, clip.clipId), 0, 1); + const plain = cut.item(track, cut.spriteId, 0, 1); + const fitted = (id: string) => + fittedLength( + cut.project, + cut.comp(), + cut.comp().items.find((i) => i.id === id)!, + ); + expect(fitted(whole)).toBe(600); + expect(fitted(trimmed)).toBe(480); + expect(fitted(short)).toBe(90); + expect(fitted(plain)).toBe(180); + expect( + itemVideo( + cut.project, + cut.comp(), + cut.comp().items.find((i) => i.id === plain)!, + ), + ).toBeNull(); + }); + + it('take the default length while the video’s length is not known yet', () => { + const cut = makeCut(); + const item = cut.item(firstTrack(cut), videoSprite(cut, addVideo(cut, 0)), 0, 1); + expect( + fittedLength( + cut.project, + cut.comp(), + cut.comp().items.find((i) => i.id === item)!, + ), + ).toBe(180); + }); + + it('follow the texture an override gives the item', () => { + const cut = makeCut(); + const item = cut.item(firstTrack(cut), cut.spriteId, 0, 1); + cut.comp().objects[0]!.overrides.texture = addVideo(cut, 4); + expect( + fittedLength( + cut.project, + cut.comp(), + cut.comp().items.find((i) => i.id === item)!, + ), + ).toBe(240); + }); +}); + +describe('saving and loading a composition', () => { + it('keeps tracks, items, clips and the objects', () => { + const cut = makeCut(); + const track = cut.track('Second'); + cut.item(track, cut.spriteId, 12, 34, { offset: 1.25, fitMedia: true }); + cut.clip(track, 'gone', 5); + const loaded = findAssetOf(roundTrip(cut.project), cut.compId, 'composition')!; + expect(loaded.tracks).toEqual(cut.comp().tracks); + expect(loaded.items).toEqual(cut.comp().items); + expect(loaded.clips).toEqual(cut.comp().clips); + expect(loaded.objects).toEqual(cut.comp().objects); + }); + + it('drops what no item spawns, an item without its object or track, and what is no time', () => { + const cut = makeCut(); + const track = firstTrack(cut); + const kept = cut.item(track, cut.spriteId, 0, 10); + cut.comp().objects.push(createSceneObject('stray', cut.spriteId, { x: 0, y: 0 })); + cut.comp().items.push({ id: 'no-object', track, start: 0, length: 10, offset: 0 }); + cut.item('no-track', cut.spriteId, 0, 10); + cut.item(track, cut.spriteId, -5, 10); + cut.item(track, cut.spriteId, 0, 0); + cut.clip('no-track', 'x', 0); + cut.clip(track, 'y', -1); + const loaded = findAssetOf(roundTrip(cut.project), cut.compId, 'composition')!; + expect(loaded.items.map((i) => i.id)).toEqual([kept]); + expect(loaded.objects.map((o) => o.id)).toEqual([kept]); + expect(loaded.clips).toEqual([]); + }); + + it('keeps every object at the top level, a negative offset at 0 and at least one track', () => { + const cut = makeCut(); + const track = firstTrack(cut); + const parent = cut.item(track, cut.spriteId, 0, 10); + const child = cut.item(track, cut.spriteId, 10, 10, { offset: -3 }); + cut.comp().objects.find((o) => o.id === child)!.parentId = parent; + const raw = structuredClone(cut.project); + findAssetOf(raw, cut.compId, 'composition')!.tracks = []; + const loaded = findAssetOf(roundTrip(raw), cut.compId, 'composition')!; + expect(loaded.tracks).toHaveLength(1); + // The one track left is a new one, so the items on the old one fall away. + expect(loaded.items).toEqual([]); + const again = findAssetOf(roundTrip(cut.project), cut.compId, 'composition')!; + expect(again.objects.map((o) => o.parentId)).toEqual(['', '']); + expect(again.items.find((i) => i.id === child)!.offset).toBe(0); + expect(again.items.some((i) => 'fitMedia' in i)).toBe(false); + }); +}); diff --git a/tools/e2e/04-assets.mjs b/tools/e2e/04-assets.mjs index a6e7b0cc..d01cbcfe 100644 --- a/tools/e2e/04-assets.mjs +++ b/tools/e2e/04-assets.mjs @@ -2,7 +2,7 @@ import { colourDistance } from '../lib/harness.mjs'; import { pickAssetType } from './lib/scaffold.mjs'; /** One per entry of `ASSET_TYPES` – a new asset type counts here too. */ -const ASSET_TYPE_COUNT = 32; +const ASSET_TYPE_COUNT = 33; /** The categories of the asset panel and the `+` menu, in their order (`ASSET_CATEGORIES`). */ const CATEGORIES = ['Basics', 'Visuals', 'Logics', 'Physics', 'Audio'];