From 6e6b6a4d9a7f2637abab5a3295ca056b571bc506 Mon Sep 17 00:00:00 2001 From: MelekTaus Date: Mon, 12 Oct 2026 10:38:19 +0200 Subject: [PATCH] Wait for a composition's videos and copy and paste its items (#417) A composition lost the first frames of a video item. Its clock ran while the player was still loading, in the tab and in the game. The game only waited for videos once, when it started. A composition that a behavior starts later spawned its items after that wait. - The view now records which video sprites it left out for lack of a first picture, one entry per object. - The composition tab holds its playhead while the preview misses a video, for at most VIDEO_WAIT_MS (3 s) like the game. - CompositionSystem holds a composition's clock while one of its live items misses its video. The rest of the game keeps running. - The editor play and the published game pass their view to the run as RunServices.pictures. - VIDEO_WAIT_MS moves to core/model/video.ts, because the runtime does not import from viewport/. - Videos loaded ahead for coming items do not hold the clock. Items can now be copied and pasted inside the composition tab. - Ctrl+C keeps the timing of the copied items next to the objects. - Ctrl+V pastes objects and items in one undo step. The earliest item lands at the frame under the pointer, on the track there. - Without the pointer over the tracks it lands at the playhead, on the selected track or on the top track of the selected items. - Several items keep their distances and tracks to each other. - The same copy pastes into a scene as plain objects. Objects copied from a scene do not paste into a composition, they have no timing. Tests: - Unit composition-edit, clipboard, composition-play: 3 new for pasting, 1 new for the clock waiting in the game - e2e 69i (play from the end waits for the video), 69j (paste under the pointer and at the playhead), 68d (a composition started from On start waits for its video) - each new check is red without its fix - README: 2036 editor checks Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- src/core/commands/compositionCommands.ts | 35 ++++++++- src/core/model/clipboard.ts | 5 ++ src/core/model/compositionEdit.ts | 27 +++++++ src/core/model/video.ts | 6 ++ src/editor/Editor.ts | 2 + src/editor/PlaySession.ts | 4 +- src/editor/clipboard.ts | 2 +- src/editor/composition/CompositionEditor.ts | 21 +++++- src/editor/composition/CompositionTimeline.ts | 50 ++++++++++++- src/player/Player.ts | 4 +- src/runtime/GameRun.ts | 3 + src/runtime/systems/BehaviorSystem.ts | 7 +- src/runtime/systems/CompositionSystem.ts | 36 ++++++++-- src/viewport/VideoLibrary.ts | 2 - src/viewport/WorldViewport.ts | 17 +++++ tests/core/composition-edit.test.ts | 47 ++++++++++++ tests/core/composition-play.test.ts | 39 +++++++++- tools/e2e/68-composition.mjs | 60 +++++++++++++++- tools/e2e/69-composition-tab.mjs | 71 ++++++++++++++++++- 20 files changed, 421 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index ec44546b..75b6d2b8 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh The browser suite and the three example games need a machine with a GPU: ```bash -node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2033 checks (GM_STEPS=20 for one step) +node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2036 checks (GM_STEPS=20 for one step) # every tool signs in first – see the Accounts page node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs diff --git a/src/core/commands/compositionCommands.ts b/src/core/commands/compositionCommands.ts index 4a7e8c43..a78b79cb 100644 --- a/src/core/commands/compositionCommands.ts +++ b/src/core/commands/compositionCommands.ts @@ -9,10 +9,12 @@ import { heldCompositionItems, heldCompositionTracks, } from '../model/composition'; -import { mediaKey } from '../model/compositionEdit'; +import type { ObjectClip } from '../model/clipboard'; +import { mediaKey, pastedItems } from '../model/compositionEdit'; import type { Project } from '../model/Project'; import type { Command } from './Command'; import { oneStep } from './CompoundCommand'; +import { PasteObjectsCommand } from './pasteCommands'; import { requireComposition } from './require'; import { PlaceAssetCommand, RemoveSceneObjectsCommand } from './sceneObjectCommands'; @@ -80,6 +82,37 @@ export function addCompositionItem( return { command: oneStep('Add item', [place, set]), objectId: place.objectId }; } +/** + * Copied items put back into a composition: their objects where the copied ones stood, the items + * timed as `pastedItems` says. Null when the copy holds anything but items – a scene's objects have + * no time to land at. `itemIds` are the new items', to pick them afterwards. + */ +export function pasteCompositionItems( + project: Project, + assetId: string, + clip: ObjectClip, + at: number, + row?: number, +): { command: Command; itemIds: string[] } | null { + const copied = new Map((clip.items ?? []).map((i) => [i.id, i])); + if (clip.roots.length === 0 || clip.roots.some((r) => !copied.has(r.id))) return null; + const comp = requireComposition(project, assetId); + const paste = new PasteObjectsCommand(assetId, clip, {}, { x: 0, y: 0 }); + const ids = new Map(clip.roots.map((r, n) => [r.id, paste.rootIds[n]!])); + const items = pastedItems( + clip.roots.map((r) => copied.get(r.id)!), + ids, + at, + comp.tracks.map((t) => t.id), + row, + ); + const set = new SetCompositionCommand(assetId, { items: [...comp.items, ...items] }); + return { + command: oneStep(items.length === 1 ? 'Paste item' : 'Paste items', [paste, set]), + itemIds: items.map((i) => i.id), + }; +} + /** Items and clips taken off together, the items' objects with them. */ export function removeFromComposition( project: Project, diff --git a/src/core/model/clipboard.ts b/src/core/model/clipboard.ts index 66536a7e..1dd42cea 100644 --- a/src/core/model/clipboard.ts +++ b/src/core/model/clipboard.ts @@ -14,6 +14,7 @@ import { import { editorTransform, parentTransform, positionOf } from '../properties/transform'; import { findObject, subtreeIds, subtreeRoots, treeOrder } from './hierarchy'; import { isLiveInstance, isTemplateContent, resolvedHolder, splitInstanceId } from './instance'; +import type { CompositionItem } from './Asset'; import type { Mat } from './matrix'; import type { Position } from './Properties'; import type { ObjectHolder, Project } from './Project'; @@ -34,6 +35,8 @@ export interface ObjectClip { /** Parents before children; a root's `parentId` is the one it had where it was copied. */ objects: SceneObject[]; roots: ClipRoot[]; + /** Copied out of a composition (#417): the items of the roots that are items, for a paste there. */ + items?: CompositionItem[]; } export interface GraphClip { @@ -76,9 +79,11 @@ export function copyObjects( .map((entry) => entry.object) .filter((o) => members.has(o.id) && !shownByInstance(o.id)) .map((o) => ({ ...structuredClone(o), editorOffset: { x: 0, y: 0 } })); + const items = holder.type === 'composition' ? holder.items.filter((i) => roots.includes(i.id)) : []; return { kind: 'objects', objects, + ...(items.length > 0 && { items: structuredClone(items) }), roots: roots.map((id) => { const object = findObject(view, id)!; return { diff --git a/src/core/model/compositionEdit.ts b/src/core/model/compositionEdit.ts index af5bb37b..cb708143 100644 --- a/src/core/model/compositionEdit.ts +++ b/src/core/model/compositionEdit.ts @@ -124,6 +124,33 @@ export function movedClips( return out && apart(out, lengthOf) ? out : null; } +/** + * Copied items under the ids `ids` gives them, moved so the earliest starts at `at`: on their tracks, + * or, given `row`, the topmost on that track and the others as far below as they were, as far as + * there are tracks – a song's paste. An item whose track is gone counts as on the first. + */ +export function pastedItems( + items: readonly CompositionItem[], + ids: ReadonlyMap, + at: number, + trackIds: readonly string[], + row?: number, +): CompositionItem[] { + if (items.length === 0) return []; + const first = Math.min(...items.map((i) => i.start)); + const index = new Map(trackIds.map((id, i) => [id, i])); + const rows = items.map((i) => index.get(i.track) ?? 0); + const top = Math.min(...rows); + const bottom = Math.max(...rows); + const shift = row === undefined ? 0 : Math.max(-top, Math.min(trackIds.length - 1 - bottom, row - top)); + return items.map((i, n) => ({ + ...i, + id: ids.get(i.id) ?? i.id, + start: Math.max(0, Math.round(at)) + (i.start - first), + track: trackIds[rows[n]! + shift] ?? i.track, + })); +} + /** The item with its end edge dragged to make it `wanted` frames long, at least one; its length is its own from then on. */ export function resizedItem(item: CompositionItem, wanted: number): CompositionItem { const { fitMedia: _, ...rest } = item; diff --git a/src/core/model/video.ts b/src/core/model/video.ts index a7fae6b2..a57ffad5 100644 --- a/src/core/model/video.ts +++ b/src/core/model/video.ts @@ -6,6 +6,12 @@ import { storedFilePattern } from './storedFile'; const FILE_PATTERN = storedFilePattern('mp4'); +/** + * How long a run waits for a driven video (#404) it shows to have its first picture before it goes + * anyway: at its start, and a composition (#417) at each item that brings one in. + */ +export const VIDEO_WAIT_MS = 3000; + export function isVideoFile(name: string): boolean { return FILE_PATTERN.test(name); } diff --git a/src/editor/Editor.ts b/src/editor/Editor.ts index 8e4b43b8..060869bf 100644 --- a/src/editor/Editor.ts +++ b/src/editor/Editor.ts @@ -768,6 +768,8 @@ export class Editor implements PlayHost { return this.viewport.pasteHere(); case 'container': return this.containerEditor.canvas.pasteHere(); + case 'composition': + return this.compositionEditor.canvas.paste(); case 'graph': return this.graphEditor.canvas.paste(); case 'statemachine': diff --git a/src/editor/PlaySession.ts b/src/editor/PlaySession.ts index e397afce..9d658df2 100644 --- a/src/editor/PlaySession.ts +++ b/src/editor/PlaySession.ts @@ -10,6 +10,7 @@ import { AnimationSystem } from '../runtime/systems/AnimationSystem'; import { type BehaviorSystem, behaviorError, type Trace } from '../runtime/systems/BehaviorSystem'; import type { AnimationPlayer } from '../core/model/animation'; import type { PropertyBag } from '../core/model/Properties'; +import { VIDEO_WAIT_MS } from '../core/model/video'; import { gameClock } from '../core/time'; import { loadPhysics, readyPhysics } from '../runtime/physics/engine'; import { Input } from '../runtime/systems/Input'; @@ -17,7 +18,7 @@ import type { World } from '../runtime/World'; import { onFrame, ui } from '../ui'; import { soundLibrary } from '../viewport/SoundLibrary'; import { GameRecorder, type MadeRecording } from '../viewport/GameRecorder'; -import { VIDEO_WAIT_MS, videoLibrary } from '../viewport/VideoLibrary'; +import { videoLibrary } from '../viewport/VideoLibrary'; import { FrameMeter, type FrameTimes } from './frameMeter'; import { saveArea } from './saves'; import { PanelClock, panelPolicy, type PanelUpdate } from './playThrottle'; @@ -188,6 +189,7 @@ export class PlaySession { clock: gameClock, saves: saveArea(runtime.id, 'play'), recorder: this.recorder, + pictures: { missing: (id) => this.host.gameView.missesVideoOf(id) }, }; this.run = GameRun.start(runtime, scene.id, this.input, services); videoLibrary.startRun(); diff --git a/src/editor/clipboard.ts b/src/editor/clipboard.ts index c17b2862..66499250 100644 --- a/src/editor/clipboard.ts +++ b/src/editor/clipboard.ts @@ -56,7 +56,7 @@ export function copyObjectIds(state: EditorState, holderId: string, ids: readonl const clip = holder && copyObjects(state.project, holder, ids); if (!clip) return false; const roots = clip.roots.map((r) => r.id); - return put(state, clip, roots, counted(roots.length, 'object')); + return put(state, clip, roots, counted(roots.length, clip.items ? 'item' : 'object')); } /** Copies the selected objects, or the selected nodes and boxes; false when nothing of that is selected. */ diff --git a/src/editor/composition/CompositionEditor.ts b/src/editor/composition/CompositionEditor.ts index 7e98c2c2..72ff98f3 100644 --- a/src/editor/composition/CompositionEditor.ts +++ b/src/editor/composition/CompositionEditor.ts @@ -1,5 +1,6 @@ import type { Asset, CompositionAsset } from '../../core/model/Asset'; import { compositionFps, compositionLength } from '../../core/model/composition'; +import { VIDEO_WAIT_MS } from '../../core/model/video'; import { Button, formatNumber, VStack, type Widget } from '../../ui'; import { AssetTabEditor } from '../AssetTabEditor'; import type { EditorState } from '../state'; @@ -87,10 +88,28 @@ export class CompositionEditor extends AssetTabEditor= 0 ? picked : undefined); + const at = spot?.frame ?? Math.floor(this.playhead + 1e-6); + const pasted = pasteCompositionItems(this.state.project, asset.id, clip, at, row); + if (!pasted || !this.state.execute(pasted.command)) return false; + this.state.select(objectSelection(asset.id, pasted.itemIds)); + return true; + } + + /** Where a paste goes with the pointer over the lanes: the frame under it and the nearest track; else null. */ + private pasteSpot(asset: CompositionAsset): { frame: number; row: number } | null { + const p = this.pointer; + if (!p || p.x < HEAD_WIDTH || p.x > this.w || p.y < RULER_HEIGHT || p.y > this.h) return null; + if (asset.tracks.length === 0) return null; + const g = this.inLanes(p); + return { + frame: Math.max(0, Math.round(xToTick(this.view, g.x))), + row: nearestTrack(this.view, asset.tracks.length, g.y), + }; + } + + /** + * The row of the picked track or of the picked clips' track, else the topmost the picked items lie + * on – the items just copied land on their own tracks again; -1 without. + */ + private pickedTrack(asset: CompositionAsset): number { + const s = this.state.selection; + const rowOf = (track: string) => asset.tracks.findIndex((t) => t.id === track); + if (s?.kind === 'composition' && s.assetId === asset.id) return rowOf(s.trackId); + const picked = new Set(this.pickedItems(asset)); + const rows = asset.items.filter((i) => picked.has(i.id)).map((i) => rowOf(i.track)); + return rows.length > 0 ? Math.min(...rows) : -1; + } + // --- test hooks -------------------------------------------------------------------------------- /** Where a track and frame lie on screen, in global coordinates: the middle of the track. */ @@ -491,7 +538,8 @@ export class CompositionTimeline extends Widget { private onPointerMove(e: FederatedPointerEvent): void { const asset = this.asset(); const local = this.toLocal(e.global); - const g = this.inLanes({ x: local.x, y: local.y }); + this.pointer = { x: local.x, y: local.y }; + const g = this.inLanes(this.pointer); const dragged = activeDrag(); if (dragged?.kind === 'asset' && asset) { this.followDrop(asset, dragged.id, local, g); diff --git a/src/player/Player.ts b/src/player/Player.ts index 6cb14a5c..54964438 100644 --- a/src/player/Player.ts +++ b/src/player/Player.ts @@ -1,5 +1,6 @@ import type { SaveArea } from '../core/graph/savedValues'; import { type Project, startScene } from '../core/model/Project'; +import { VIDEO_WAIT_MS } from '../core/model/video'; import { gameClock, timeStep } from '../core/time'; import { GameRun } from '../runtime/GameRun'; import { loadPhysics, readyPhysics } from '../runtime/physics/engine'; @@ -8,7 +9,7 @@ import { Input } from '../runtime/systems/Input'; import { downloadBlob, requestRender } from '../ui'; import { GameRecorder } from '../viewport/GameRecorder'; import { soundLibrary } from '../viewport/SoundLibrary'; -import { VIDEO_WAIT_MS, videoLibrary } from '../viewport/VideoLibrary'; +import { videoLibrary } from '../viewport/VideoLibrary'; import type { GameScreen } from './GameScreen'; /** @@ -71,6 +72,7 @@ export class Player { clock: gameClock, saves: this.saves, recorder: this.recorder, + pictures: { missing: (id) => this.screen.missesVideoOf(id) }, }); videoLibrary.startRun(); this.input.attach(); diff --git a/src/runtime/GameRun.ts b/src/runtime/GameRun.ts index 0dbb9b91..c461d8a0 100644 --- a/src/runtime/GameRun.ts +++ b/src/runtime/GameRun.ts @@ -8,6 +8,7 @@ import { findScene, type Project } from '../core/model/Project'; import { mixOf } from '../core/music/mixer'; import type { RecordingSink } from '../core/recording/recording'; import type { AudioSink } from './systems/Audio'; +import type { VideoPictures } from './systems/CompositionSystem'; import { addGameSystems, type BehaviorSystem, type TimeScaled } from './systems/BehaviorSystem'; import type { Input } from './systems/Input'; import { bringObjects, keptEntities, objectsToBring } from './keptObjects'; @@ -21,6 +22,8 @@ export interface RunServices { saves: SaveArea; /** Outlives a `Load scene` and a restart, so a recording goes on through both (#405). */ recorder?: RecordingSink; + /** The view the run is shown in, asked whether a video has its first picture yet (#417). */ + pictures?: VideoPictures; } /** What a scene holds of objects, which is all a run writes into it. */ diff --git a/src/runtime/systems/BehaviorSystem.ts b/src/runtime/systems/BehaviorSystem.ts index 6c97535d..59c66ad9 100644 --- a/src/runtime/systems/BehaviorSystem.ts +++ b/src/runtime/systems/BehaviorSystem.ts @@ -55,7 +55,7 @@ import { import { CollisionSystem } from './CollisionSystem'; import type { CollisionSource } from './solids'; import { AnimationSystem } from './AnimationSystem'; -import { CompositionSystem } from './CompositionSystem'; +import { CompositionSystem, type VideoPictures } from './CompositionSystem'; import type { Input } from './Input'; import type { RecordingSink } from '../../core/recording/recording'; import type { System } from './System'; @@ -122,6 +122,8 @@ export interface GameSystemOptions { recorder?: RecordingSink | null; /** Whether `Play composition` plays (#417); the editor run ignores it. */ playsCompositions?: boolean; + /** The view a played game is shown in: a composition waits for its videos there (#417). */ + pictures?: VideoPictures | null; } /** Collisions, compositions and animations are ticked first, so behaviors see this frame's events. */ @@ -139,10 +141,11 @@ export function addGameSystems( saves = null, recorder = null, playsCompositions = true, + pictures = null, }: GameSystemOptions = {}, ): BehaviorSystem { const collisions = physics ? new PhysicsSystem(physics) : new CollisionSystem(); - const compositions = new CompositionSystem(playsCompositions); + const compositions = new CompositionSystem(playsCompositions, pictures); const animations = new AnimationSystem(); const sources = { collisions, animations, compositions, audio, clock, saves, recorder }; const behaviors = new BehaviorSystem(project, input, sources, shouldRun); diff --git a/src/runtime/systems/CompositionSystem.ts b/src/runtime/systems/CompositionSystem.ts index 9ac4c973..4b3dcce1 100644 --- a/src/runtime/systems/CompositionSystem.ts +++ b/src/runtime/systems/CompositionSystem.ts @@ -13,7 +13,7 @@ import { curveLookup } from '../../core/model/curve'; import { contentId, placedWithContent } from '../../core/model/instance'; import { findAssetOf } from '../../core/model/Project'; import type { PropertyValue } from '../../core/model/Properties'; -import { findVideo } from '../../core/model/video'; +import { findVideo, VIDEO_WAIT_MS } from '../../core/model/video'; import { VIDEO_CLOCK_KEY, VIDEO_TIME_KEY, videoDrivenOf } from '../../core/properties/videos'; import type { Entity } from '../Entity'; import type { World } from '../World'; @@ -38,6 +38,14 @@ interface Run { id: number; /** By item id. An item a behavior destroyed stays here until it ends, so it is not built again. */ live: Map; + /** Seconds the clock has stood so far for a video without its first picture, see `waits`. */ + waited: number; +} + +/** What the view that shows a run knows of its driven videos (#404). */ +export interface VideoPictures { + /** Whether it left the sprite `objectId` out, its video still without a first picture. */ + missing(objectId: string): boolean; } /** @@ -46,12 +54,18 @@ interface Run { * videos. Runs before the animations and the behaviors, so a behavior writes last and hears this * frame's `finished`. The editor run plays none: it writes into the saved scene, where the objects of * a cut would stay behind. + * + * While the view still waits for the first picture of a video an item brings in, the composition's + * clock stands, so the video starts at its first frame and not where the clock has got to meanwhile. */ export class CompositionSystem implements System { finished: CompositionFinish[] = []; private readonly runs = new Map(); - constructor(private readonly plays = true) {} + constructor( + private readonly plays = true, + private readonly pictures: VideoPictures | null = null, + ) {} update(world: World, dt: number): void { this.finished = []; @@ -82,8 +96,8 @@ export class CompositionSystem implements System { return; } run = this.runs.get(placement); - if (!run) this.runs.set(placement, (run = { id: player.run, live: new Map() })); - if (player.shown && isHandleEnabled(placement)) player.time += dt; + if (!run) this.runs.set(placement, (run = { id: player.run, live: new Map(), waited: 0 })); + if (player.shown && isHandleEnabled(placement) && !this.waits(run, dt)) player.time += dt; player.shown = true; const length = compositionLength(world.project, comp); let frame = compositionFrameAt(comp, player.time); @@ -99,6 +113,20 @@ export class CompositionSystem implements System { this.show(world, comp, placement, run, frame); } + /** Whether a video of a live item still has no picture; at most `VIDEO_WAIT_MS` at a stretch. */ + private waits(run: Run, dt: number): boolean { + const pictures = this.pictures; + const missing = + pictures !== null && + [...run.live.values()].some(({ videos }) => videos.some((v) => pictures.missing(v.objectId))); + if (!missing) { + run.waited = 0; + return false; + } + run.waited += dt; + return run.waited * 1000 < VIDEO_WAIT_MS; + } + /** Takes every item of the placement's run away. */ private end(world: World, placement: Entity): void { const run = this.runs.get(placement); diff --git a/src/viewport/VideoLibrary.ts b/src/viewport/VideoLibrary.ts index c76f732b..19782369 100644 --- a/src/viewport/VideoLibrary.ts +++ b/src/viewport/VideoLibrary.ts @@ -40,8 +40,6 @@ const IDLE_MS = 2000; const SEEK_SLACK = 1 / 120; /** How long after `seeked` a seek counts as landed when Chrome hands over no frame for it. */ const LAND_MS = 150; -/** How long a run waits at its start for the driven videos it shows (#404) before it goes anyway. */ -export const VIDEO_WAIT_MS = 3000; /** * When the frame being drawn began, in milliseconds like `performance.now()`. The ticker moves every diff --git a/src/viewport/WorldViewport.ts b/src/viewport/WorldViewport.ts index a1fa260b..6821369e 100644 --- a/src/viewport/WorldViewport.ts +++ b/src/viewport/WorldViewport.ts @@ -125,6 +125,8 @@ export abstract class WorldViewport extends W private readonly videosShown = new Map(); /** Since when the view keeps its last frame for a driven video it brings in, see `holdsForVideo`. */ private heldSince: number | null = null; + /** The sprites the last redraw left out, their driven video still without its first picture. */ + private readonly videosLeftOut = new Set(); private cameraMoves = 0; private readonly stopFiles: (() => void)[]; /** One view per overlay camera (#302), by camera id, laid over this one in camera order. */ @@ -408,6 +410,7 @@ export abstract class WorldViewport extends W this.content.visible = view !== null; this.visibleCount = 0; this.animated = false; + this.videosLeftOut.clear(); this.drawn = null; if (!view || !frame) { this.unshadeCamera(); @@ -885,6 +888,7 @@ export abstract class WorldViewport extends W if (ghost) disposeSpriteMesh(ghost); this.ghostSprites.delete(item.id); this.videosShown.delete(item.id); + if (!videoLibrary.hasPicture(key)) this.videosLeftOut.add(item.id); return; } if (key !== null) this.videosShown.set(item.id, key); @@ -927,6 +931,19 @@ export abstract class WorldViewport extends W return videoLibrary.loading(this.wearer); } + /** + * Whether the last redraw left out a driven video (#404) it should show, its player still without + * a first picture. Unlike `videosLoading`, one opened ahead or switched off does not count. + */ + missesVideo(): boolean { + return this.videosLeftOut.size > 0; + } + + /** Whether the last redraw left out the sprite `objectId` so, see `missesVideo`. */ + missesVideoOf(objectId: string): boolean { + return this.videosLeftOut.has(objectId); + } + /** Whether a driven video (#404) drawn here may be heard: in a run, and in the animation preview. */ protected hearsVideo(): boolean { return false; diff --git a/tests/core/composition-edit.test.ts b/tests/core/composition-edit.test.ts index c492ab1a..ce1c714d 100644 --- a/tests/core/composition-edit.test.ts +++ b/tests/core/composition-edit.test.ts @@ -9,6 +9,7 @@ import { CreateAssetCommand, CreateVideoCommand } from '../../src/core/commands/ import { addCompositionItem, FitMediaCommand, + pasteCompositionItems, removeCompositionTrack, removeFromComposition, SetCompositionCommand, @@ -24,10 +25,12 @@ import { mediaMarks, movedClips, movedItems, + pastedItems, resizedItem, snappedToMedia, trimmedItem, } from '../../src/core/model/compositionEdit'; +import { copyObjects } from '../../src/core/model/clipboard'; import { hitAt, overlaps, uncoveredEnd } from '../../src/editor/composition/compositionView'; import { placementRejection } from '../../src/core/model/placement'; import { findAsset, findAssetOf } from '../../src/core/model/Project'; @@ -263,6 +266,50 @@ describe('items and their objects', () => { expect(placementRejection(cut.project, cut.comp(), findAsset(cut.project, cut.compId)!)).not.toBe(''); }); + it('pasted items keep their timing from the frame they land at, on their tracks or shifted to a row', () => { + const copied = [item('a', 't2', 30, 60, { offset: 1.5 }), item('b', 't3', 50, 20)]; + const ids = new Map([ + ['a', 'a2'], + ['b', 'b2'], + ]); + const tracks = ['t1', 't2', 't3']; + expect(pastedItems(copied, ids, 100, tracks)).toEqual([ + item('a2', 't2', 100, 60, { offset: 1.5 }), + item('b2', 't3', 120, 20), + ]); + expect(pastedItems(copied, ids, 0, tracks, 0).map((i) => i.track)).toEqual(['t1', 't2']); + // As far as there are tracks: the bottom one may not fall off. + expect(pastedItems(copied, ids, 0, tracks, 2).map((i) => i.track)).toEqual(['t2', 't3']); + }); + + it('a copied item is pasted with its object and overrides under new ids, in one undo step', () => { + const cut = makeCut(); + const id = cut.add(10, 60); + cut.stack.execute(new SetOverrideCommand(cut.compId, id, 'width', 42)); + const clip = copyObjects(cut.project, cut.comp(), [id])!; + const pasted = pasteCompositionItems(cut.project, cut.compId, clip, 100)!; + cut.stack.execute(pasted.command); + const [copy] = pasted.itemIds; + expect(copy).not.toBe(id); + expect(cut.comp().items.find((i) => i.id === copy)).toMatchObject({ + track: cut.track, + start: 100, + length: 60, + }); + expect(cut.comp().objects.find((o) => o.id === copy)?.overrides).toMatchObject({ width: 42 }); + cut.stack.undo(); + expect(cut.comp().items.map((i) => i.id)).toEqual([id]); + expect(cut.comp().objects.map((o) => o.id)).toEqual([id]); + }); + + it('objects without their items, as a scene copies them, are not pasted into a composition', () => { + const cut = makeCut(); + const id = cut.add(0, 60); + const { items, ...plain } = copyObjects(cut.project, cut.comp(), [id])!; + expect(items).toHaveLength(1); + expect(pasteCompositionItems(cut.project, cut.compId, plain, 0)).toBeNull(); + }); + it('a sprite wearing a video is dropped in as long as the video, anything else 3 s', () => { const cut = makeCut(); const screen = new CreateAssetCommand('sprite', 'Screen', { texture: cut.videoId }); diff --git a/tests/core/composition-play.test.ts b/tests/core/composition-play.test.ts index c5fd3694..a4d63335 100644 --- a/tests/core/composition-play.test.ts +++ b/tests/core/composition-play.test.ts @@ -21,6 +21,7 @@ import { findAssetOf, findScene } from '../../src/core/model/Project'; import { createSceneObject } from '../../src/core/model/SceneObject'; import { VIDEO_CLOCK_KEY, VIDEO_TIME_KEY } from '../../src/core/properties/videos'; import { addGameSystems } from '../../src/runtime/systems/BehaviorSystem'; +import type { VideoPictures } from '../../src/runtime/systems/CompositionSystem'; import { Input } from '../../src/runtime/systems/Input'; import { World } from '../../src/runtime/World'; import { stackAt } from '../../src/viewport/pick'; @@ -105,9 +106,9 @@ function director(stage: Stage): void { } /** The scene played as a game: `frame()` ticks a tenth of a second and writes back like Play does. */ -function play(stage: Stage, playsCompositions = true) { +function play(stage: Stage, playsCompositions = true, pictures: VideoPictures | null = null) { const world = World.instantiate(stage.project, stage.sceneId); - addGameSystems(world, stage.project, new Input(), { playsCompositions }); + addGameSystems(world, stage.project, new Input(), { playsCompositions, pictures }); const placement = () => world.findByObjectId(stage.placement)!; const at = (item: string) => world.findByObjectId(contentId(stage.placement, item)); return { @@ -224,6 +225,40 @@ describe('a composition played in a scene', () => { expect(run.at(item)!.props[VIDEO_TIME_KEY]).toBeCloseTo(2, 9); }); + it('stands still while the view waits for the first picture of an item’s video, 3 s at most', () => { + const stage = makeStage(); + director(stage); + const video = new CreateVideoCommand('Intro', FILE, { + width: 320, + height: 180, + duration: 10, + fps: 30, + audio: false, + }); + stage.stack.execute(video); + const screen = new CreateAssetCommand('sprite', 'Screen', { texture: video.asset.id }); + stage.stack.execute(screen); + const item = stage.item(first(stage), screen.asset.id, 0, 100); + const missing = new Set(); + const run = play(stage, true, { missing: (id) => missing.has(id) }); + const time = () => run.placement().composition!.time; + run.frame(); + // The view drew the new item without its picture. + missing.add(contentId(stage.placement, item)); + run.frame(5); + expect(time()).toBe(0); + expect(run.at(item)!.props[VIDEO_TIME_KEY]).toBe(0); + missing.clear(); + run.frame(2); + expect(time()).toBeCloseTo(0.2, 9); + // A picture that never comes holds it 3 s, then it goes on without. + missing.add(contentId(stage.placement, item)); + run.frame(29); + expect(time()).toBeCloseTo(0.2, 9); + run.frame(10); + expect(time()).toBeGreaterThan(1); + }); + it('hands the view the video of the next item a second ahead, by the id the item will take', () => { const stage = makeStage(); director(stage); diff --git a/tools/e2e/68-composition.mjs b/tools/e2e/68-composition.mjs index 4a9e1843..07fbd9a4 100644 --- a/tools/e2e/68-composition.mjs +++ b/tools/e2e/68-composition.mjs @@ -1,3 +1,5 @@ +import { readFileSync } from 'node:fs'; +import { uploadVideo } from '../lib/projects.mjs'; import { MAKE_OBJECT } from './lib/scaffold.mjs'; /** @@ -5,7 +7,8 @@ import { MAKE_OBJECT } from './lib/scaffold.mjs'; * first frame, and a click on that picture picks the placement. Played, a behavior below it starts * it: each item becomes an object below the placement while it lasts, the first track in front, a * clip moves the item it overlaps, and `On composition finished` hears the end, by which time no - * item is left. Nothing of the run stays in the scene. + * item is left. Nothing of the run stays in the scene. An item's video is waited for: the clock + * stands until the Game tab has its first picture. * * Everything it adds carries a `cp68` id and goes again at the end. */ @@ -161,6 +164,61 @@ export async function composition(t) { JSON.stringify(stored), ); + // 68d. A video item at frame 0, played from On start: the composition's clock stands while the Game + // tab loads the video, so its first frames are seen – it ran on without them before. + const uploaded = await uploadVideo( + t.baseUrl, + readFileSync(new URL('./fixtures/colors.webm', import.meta.url)), + 'colors.webm', + ); + await mutate(` + const p = __gm.state.project; + p.assets.push( + { id: 'cp68-video', type: 'video', name: 'Colors 68', props: {}, file: ${JSON.stringify(uploaded.file)}, + width: 320, height: 180, duration: 2, fps: 30, audio: true }, + { id: 'cp68-screen', type: 'sprite', name: 'Screen 68', props: { width: 160, height: 90, color: 0xffffff, texture: 'cp68-video' } }, + ); + const c = p.assets.find((a) => a.id === 'cp68-cut'); + // Sixty frames a second: in steps of a tenth, a clock that ran on for a few frames still read 0. + c.props.fps = 60; + c.objects = [{ ...structuredClone(c.objects[0]), id: 'cp68-v', assetId: 'cp68-screen', name: 'Video 68', overrides: {} }]; + c.items = [{ id: 'cp68-v', track: 'cp68-front', start: 0, length: 120, offset: 0 }]; + c.clips = [];`); + await play(); + // The video time the composition writes into its item every frame. + const clock = `__gm.world()?.entities.find((e) => e.objectId === '${id('v')}')?.props.videoTime ?? null`; + const wait = JSON.parse( + await evaluate(`new Promise((done) => { + const loading = []; + const step = () => { + const time = ${clock}; + const player = __gm.videos().find((v) => v.key.endsWith(':${id('v')}')); + if (player?.ready) return done(JSON.stringify({ loading, ready: time })); + if (player) loading.push(time); + if (!__gm.world() || loading.length > 300) return done(JSON.stringify({ loading, ready: null })); + requestAnimationFrame(step); + }; + step(); + })`), + ); + await sleep(500); + const after = await json(clock); + check( + 'a played composition waits for the first picture of an item’s video, then runs on', + wait.loading.length > 0 && + wait.loading.every((time) => time === 0) && + wait.ready !== null && + wait.ready === 0 && + after > 0.2, + JSON.stringify({ + frames: wait.loading.length, + loading: wait.loading.slice(0, 8), + ready: wait.ready, + after, + }), + ); + await stop(); + await mutate(` const p = __gm.state.project; p.assets = p.assets.filter((a) => !a.id.startsWith('cp68-')); diff --git a/tools/e2e/69-composition-tab.mjs b/tools/e2e/69-composition-tab.mjs index 7593c722..d189409e 100644 --- a/tools/e2e/69-composition-tab.mjs +++ b/tools/e2e/69-composition-tab.mjs @@ -4,7 +4,8 @@ * over it. The end of the first, covered by the second, lights up under the pointer and is dragged * past it. An animation dragged in becomes a clip that moves the item it overlaps in the picture. The hierarchy lists what is alive at the playhead, Delete * takes an item with its object, and Play runs the playhead to the end. The video of an item about - * to begin is opened ahead, so the cut to it shows its picture at once. + * to begin is opened ahead, so the cut to it shows its picture at once, and Play waits for a video + * whose player is still loading. A copied item is pasted under the pointer or at the playhead. * * Everything it adds carries a `ct69` id and goes again at the end. */ @@ -239,6 +240,74 @@ export async function compositionTab(t) { JSON.stringify({ early, ahead, cut }), ); + // 69i. Play at the end starts over at 0, where the item's video was never opened: the playhead + // waits for its first picture, so its first frames are seen – it ran on without them before. + await mutate(` + const c = __gm.state.project.assets.find((a) => a.id === '${CUT}'); + c.objects = [{ ...structuredClone(c.objects[0]), id: 'ct69-v3' }]; + c.items = [{ id: 'ct69-v3', track: 'ct69-t1', start: 0, length: 20, offset: 0 }];`); + await evaluate(`__gm.composition.scrub(20)`); + await sleep(200); + await mouse('mousePressed', playAt.x, playAt.y); + await mouse('mouseReleased', playAt.x, playAt.y); + const wait = JSON.parse( + await evaluate(`new Promise((done) => { + const loading = []; + const step = () => { + const player = __gm.videos().find((v) => v.key.endsWith(':ct69-v3')); + const { playhead } = __gm.composition.view(); + if (player?.ready) return done(JSON.stringify({ loading, ready: playhead })); + if (player) loading.push(playhead); + if (loading.length > 300) return done(JSON.stringify({ loading, ready: null })); + requestAnimationFrame(step); + }; + step(); + })`), + ); + await sleep(500); + const after = await json('__gm.composition.view()'); + check( + 'Play waits at a video item whose player is still loading, then runs on', + wait.loading.length > 0 && + wait.loading.every((at) => at === 0) && + wait.ready !== null && + wait.ready < 0.5 && + after.playhead > 2, + JSON.stringify({ frames: wait.loading.length, ready: wait.ready, after: after.playhead }), + ); + + // 69j. Ctrl+C copies the picked item. Ctrl+V pastes it under the pointer while that is over the + // lanes, on the track there; with the pointer off the lanes at the playhead, on the picked track. + for (let i = 0; i < 30 && (await json('__gm.composition.view()')).playing; i++) await sleep(150); + await mutate(` + const c = __gm.state.project.assets.find((a) => a.id === '${CUT}'); + c.objects = [{ ...structuredClone(c.objects[0]), id: 'ct69-p1', assetId: '${BOX}' }]; + c.items = [{ id: 'ct69-p1', track: 'ct69-t1', start: 0, length: 30, offset: 0 }];`); + const original = await json(`__gm.composition.box('ct69-p1')`); + await t.click(original.x + original.w / 2, original.y + original.h / 2); + await press('c', { ctrl: true }); + const lane = await json(`__gm.composition.laneAt('ct69-t2', 50)`); + await mouse('mouseMoved', lane.x, lane.y); + await press('v', { ctrl: true }); + await evaluate(`__gm.composition.scrub(12)`); + await mouse('mouseMoved', playAt.x, playAt.y); + await press('v', { ctrl: true }); + m = await model(); + const copies = m.items.filter((i) => i.id !== 'ct69-p1'); + const pickedNow = await json('__gm.state.selection'); + check( + 'Ctrl+V pastes a copied item under the pointer, else at the playhead on the picked track', + m.items.length === 3 && + m.objects.length === 3 && + copies[0]?.track === 'ct69-t2' && + copies[0].start === 50 && + copies[0].length === 30 && + copies[1]?.track === 'ct69-t2' && + copies[1].start === 12 && + pickedNow?.objectIds?.[0] === copies[1].id, + JSON.stringify({ copies, picked: pickedNow?.objectIds }), + ); + await showTab('scene'); await mutate(` const p = __gm.state.project;