Wait for a composition's videos and copy and paste its items (#417)

A composition lost the first frames of a video item. Its clock ran
while the player was still loading, in the tab and in the game. The
game only waited for videos once, when it started. A composition that
a behavior starts later spawned its items after that wait.

- The view now records which video sprites it left out for lack of a
  first picture, one entry per object.
- The composition tab holds its playhead while the preview misses a
  video, for at most VIDEO_WAIT_MS (3 s) like the game.
- CompositionSystem holds a composition's clock while one of its live
  items misses its video. The rest of the game keeps running.
- The editor play and the published game pass their view to the run
  as RunServices.pictures.
- VIDEO_WAIT_MS moves to core/model/video.ts, because the runtime
  does not import from viewport/.
- Videos loaded ahead for coming items do not hold the clock.

Items can now be copied and pasted inside the composition tab.

- Ctrl+C keeps the timing of the copied items next to the objects.
- Ctrl+V pastes objects and items in one undo step. The earliest item
  lands at the frame under the pointer, on the track there.
- Without the pointer over the tracks it lands at the playhead, on the
  selected track or on the top track of the selected items.
- Several items keep their distances and tracks to each other.
- The same copy pastes into a scene as plain objects. Objects copied
  from a scene do not paste into a composition, they have no timing.

Tests:
- Unit composition-edit, clipboard, composition-play: 3 new for
  pasting, 1 new for the clock waiting in the game
- e2e 69i (play from the end waits for the video), 69j (paste under
  the pointer and at the playhead), 68d (a composition started from
  On start waits for its video)
- each new check is red without its fix
- README: 2036 editor checks

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/core/commands/compositionCommands.ts

@@ -9,10 +9,12 @@ import {
   heldCompositionItems,
   heldCompositionTracks,
 } from '../model/composition';
-import { mediaKey } from '../model/compositionEdit';
+import type { ObjectClip } from '../model/clipboard';
+import { mediaKey, pastedItems } from '../model/compositionEdit';
 import type { Project } from '../model/Project';
 import type { Command } from './Command';
 import { oneStep } from './CompoundCommand';
+import { PasteObjectsCommand } from './pasteCommands';
 import { requireComposition } from './require';
 import { PlaceAssetCommand, RemoveSceneObjectsCommand } from './sceneObjectCommands';
 
@@ -80,6 +82,37 @@ export function addCompositionItem(
   return { command: oneStep('Add item', [place, set]), objectId: place.objectId };
 }
 
+/**
+ * Copied items put back into a composition: their objects where the copied ones stood, the items
+ * timed as `pastedItems` says. Null when the copy holds anything but items – a scene's objects have
+ * no time to land at. `itemIds` are the new items', to pick them afterwards.
+ */
+export function pasteCompositionItems(
+  project: Project,
+  assetId: string,
+  clip: ObjectClip,
+  at: number,
+  row?: number,
+): { command: Command; itemIds: string[] } | null {
+  const copied = new Map((clip.items ?? []).map((i) => [i.id, i]));
+  if (clip.roots.length === 0 || clip.roots.some((r) => !copied.has(r.id))) return null;
+  const comp = requireComposition(project, assetId);
+  const paste = new PasteObjectsCommand(assetId, clip, {}, { x: 0, y: 0 });
+  const ids = new Map(clip.roots.map((r, n) => [r.id, paste.rootIds[n]!]));
+  const items = pastedItems(
+    clip.roots.map((r) => copied.get(r.id)!),
+    ids,
+    at,
+    comp.tracks.map((t) => t.id),
+    row,
+  );
+  const set = new SetCompositionCommand(assetId, { items: [...comp.items, ...items] });
+  return {
+    command: oneStep(items.length === 1 ? 'Paste item' : 'Paste items', [paste, set]),
+    itemIds: items.map((i) => i.id),
+  };
+}
+
 /** Items and clips taken off together, the items' objects with them. */
 export function removeFromComposition(
   project: Project,

src/core/model/clipboard.ts

@@ -14,6 +14,7 @@ import {
 import { editorTransform, parentTransform, positionOf } from '../properties/transform';
 import { findObject, subtreeIds, subtreeRoots, treeOrder } from './hierarchy';
 import { isLiveInstance, isTemplateContent, resolvedHolder, splitInstanceId } from './instance';
+import type { CompositionItem } from './Asset';
 import type { Mat } from './matrix';
 import type { Position } from './Properties';
 import type { ObjectHolder, Project } from './Project';
@@ -34,6 +35,8 @@ export interface ObjectClip {
   /** Parents before children; a root's `parentId` is the one it had where it was copied. */
   objects: SceneObject[];
   roots: ClipRoot[];
+  /** Copied out of a composition (#417): the items of the roots that are items, for a paste there. */
+  items?: CompositionItem[];
 }
 
 export interface GraphClip {
@@ -76,9 +79,11 @@ export function copyObjects(
     .map((entry) => entry.object)
     .filter((o) => members.has(o.id) && !shownByInstance(o.id))
     .map((o) => ({ ...structuredClone(o), editorOffset: { x: 0, y: 0 } }));
+  const items = holder.type === 'composition' ? holder.items.filter((i) => roots.includes(i.id)) : [];
   return {
     kind: 'objects',
     objects,
+    ...(items.length > 0 && { items: structuredClone(items) }),
     roots: roots.map((id) => {
       const object = findObject(view, id)!;
       return {

src/core/model/compositionEdit.ts

@@ -124,6 +124,33 @@ export function movedClips(
   return out && apart(out, lengthOf) ? out : null;
 }
 
+/**
+ * Copied items under the ids `ids` gives them, moved so the earliest starts at `at`: on their tracks,
+ * or, given `row`, the topmost on that track and the others as far below as they were, as far as
+ * there are tracks – a song's paste. An item whose track is gone counts as on the first.
+ */
+export function pastedItems(
+  items: readonly CompositionItem[],
+  ids: ReadonlyMap<string, string>,
+  at: number,
+  trackIds: readonly string[],
+  row?: number,
+): CompositionItem[] {
+  if (items.length === 0) return [];
+  const first = Math.min(...items.map((i) => i.start));
+  const index = new Map(trackIds.map((id, i) => [id, i]));
+  const rows = items.map((i) => index.get(i.track) ?? 0);
+  const top = Math.min(...rows);
+  const bottom = Math.max(...rows);
+  const shift = row === undefined ? 0 : Math.max(-top, Math.min(trackIds.length - 1 - bottom, row - top));
+  return items.map((i, n) => ({
+    ...i,
+    id: ids.get(i.id) ?? i.id,
+    start: Math.max(0, Math.round(at)) + (i.start - first),
+    track: trackIds[rows[n]! + shift] ?? i.track,
+  }));
+}
+
 /** The item with its end edge dragged to make it `wanted` frames long, at least one; its length is its own from then on. */
 export function resizedItem(item: CompositionItem, wanted: number): CompositionItem {
   const { fitMedia: _, ...rest } = item;

src/core/model/video.ts

@@ -6,6 +6,12 @@ import { storedFilePattern } from './storedFile';
 
 const FILE_PATTERN = storedFilePattern('mp4');
 
+/**
+ * How long a run waits for a driven video (#404) it shows to have its first picture before it goes
+ * anyway: at its start, and a composition (#417) at each item that brings one in.
+ */
+export const VIDEO_WAIT_MS = 3000;
+
 export function isVideoFile(name: string): boolean {
   return FILE_PATTERN.test(name);
 }

src/editor/Editor.ts

@@ -768,6 +768,8 @@ export class Editor implements PlayHost {
         return this.viewport.pasteHere();
       case 'container':
         return this.containerEditor.canvas.pasteHere();
+      case 'composition':
+        return this.compositionEditor.canvas.paste();
       case 'graph':
         return this.graphEditor.canvas.paste();
       case 'statemachine':

src/editor/PlaySession.ts

@@ -10,6 +10,7 @@ import { AnimationSystem } from '../runtime/systems/AnimationSystem';
 import { type BehaviorSystem, behaviorError, type Trace } from '../runtime/systems/BehaviorSystem';
 import type { AnimationPlayer } from '../core/model/animation';
 import type { PropertyBag } from '../core/model/Properties';
+import { VIDEO_WAIT_MS } from '../core/model/video';
 import { gameClock } from '../core/time';
 import { loadPhysics, readyPhysics } from '../runtime/physics/engine';
 import { Input } from '../runtime/systems/Input';
@@ -17,7 +18,7 @@ import type { World } from '../runtime/World';
 import { onFrame, ui } from '../ui';
 import { soundLibrary } from '../viewport/SoundLibrary';
 import { GameRecorder, type MadeRecording } from '../viewport/GameRecorder';
-import { VIDEO_WAIT_MS, videoLibrary } from '../viewport/VideoLibrary';
+import { videoLibrary } from '../viewport/VideoLibrary';
 import { FrameMeter, type FrameTimes } from './frameMeter';
 import { saveArea } from './saves';
 import { PanelClock, panelPolicy, type PanelUpdate } from './playThrottle';
@@ -188,6 +189,7 @@ export class PlaySession {
       clock: gameClock,
       saves: saveArea(runtime.id, 'play'),
       recorder: this.recorder,
+      pictures: { missing: (id) => this.host.gameView.missesVideoOf(id) },
     };
     this.run = GameRun.start(runtime, scene.id, this.input, services);
     videoLibrary.startRun();

src/editor/clipboard.ts

@@ -56,7 +56,7 @@ export function copyObjectIds(state: EditorState, holderId: string, ids: readonl
   const clip = holder && copyObjects(state.project, holder, ids);
   if (!clip) return false;
   const roots = clip.roots.map((r) => r.id);
-  return put(state, clip, roots, counted(roots.length, 'object'));
+  return put(state, clip, roots, counted(roots.length, clip.items ? 'item' : 'object'));
 }
 
 /** Copies the selected objects, or the selected nodes and boxes; false when nothing of that is selected. */

src/editor/composition/CompositionEditor.ts

@@ -1,5 +1,6 @@
 import type { Asset, CompositionAsset } from '../../core/model/Asset';
 import { compositionFps, compositionLength } from '../../core/model/composition';
+import { VIDEO_WAIT_MS } from '../../core/model/video';
 import { Button, formatNumber, VStack, type Widget } from '../../ui';
 import { AssetTabEditor } from '../AssetTabEditor';
 import type { EditorState } from '../state';
@@ -87,10 +88,28 @@ export class CompositionEditor extends AssetTabEditor<CompositionAsset, Composit
   /** Only called while the tab is visible. */
   tick(dt: number): void {
     const drawn = this.preview.redrawCount;
-    if (this.canvas.isPlaying) this.canvas.tick(dt);
+    if (this.canvas.isPlaying && !this.waitsForVideo()) this.canvas.tick(dt);
     this.preview.animate(drawn);
   }
 
+  /** Until when the playhead waits for a video it shows, see `waitsForVideo`. */
+  private videoWait: number | null = null;
+
+  /**
+   * Whether the playhead stands while the picture still lacks a video it shows, as a run waits at
+   * its start: an item whose player opened only now played on without its first frames. At most
+   * `VIDEO_WAIT_MS` at a time.
+   */
+  private waitsForVideo(): boolean {
+    if (!this.preview.missesVideo()) {
+      this.videoWait = null;
+      return false;
+    }
+    const now = performance.now();
+    this.videoWait ??= now + VIDEO_WAIT_MS;
+    return now < this.videoWait;
+  }
+
   get isPlaying(): boolean {
     return this.canvas.isPlaying;
   }

src/editor/composition/CompositionTimeline.ts

@@ -8,6 +8,7 @@ import {
 } from 'pixi.js';
 import {
   addCompositionItem,
+  pasteCompositionItems,
   removeCompositionTrack,
   removeFromComposition,
   SetCompositionCommand,
@@ -168,6 +169,8 @@ export class CompositionTimeline extends Widget {
   private drop: Drop | null = null;
   /** The end of an item the pointer would drag; it lights up, and the cursor says so. */
   private hover: { id: string; part: Part } | null = null;
+  /** Where the pointer was last, in local coordinates: a paste lands there while it is over the lanes. */
+  private pointer: Vec2 | null = null;
   private needsFit = true;
   /** Where the playhead stands, in frames – fractional while playing. */
   playhead = 0;
@@ -310,6 +313,50 @@ export class CompositionTimeline extends Widget {
     this.onPlayhead();
   }
 
+  /**
+   * Puts copied items in (Ctrl+V): the earliest under the pointer while it is over the lanes, the
+   * topmost on its track; else at the playhead, on the picked track or the track of the picked item,
+   * or on the tracks they came from. False when the copy is not items of a composition.
+   */
+  paste(): boolean {
+    const asset = this.asset();
+    const clip = this.state.clipboard?.clip;
+    if (!asset || clip?.kind !== 'objects') return false;
+    const spot = this.pasteSpot(asset);
+    const picked = this.pickedTrack(asset);
+    const row = spot?.row ?? (picked >= 0 ? picked : undefined);
+    const at = spot?.frame ?? Math.floor(this.playhead + 1e-6);
+    const pasted = pasteCompositionItems(this.state.project, asset.id, clip, at, row);
+    if (!pasted || !this.state.execute(pasted.command)) return false;
+    this.state.select(objectSelection(asset.id, pasted.itemIds));
+    return true;
+  }
+
+  /** Where a paste goes with the pointer over the lanes: the frame under it and the nearest track; else null. */
+  private pasteSpot(asset: CompositionAsset): { frame: number; row: number } | null {
+    const p = this.pointer;
+    if (!p || p.x < HEAD_WIDTH || p.x > this.w || p.y < RULER_HEIGHT || p.y > this.h) return null;
+    if (asset.tracks.length === 0) return null;
+    const g = this.inLanes(p);
+    return {
+      frame: Math.max(0, Math.round(xToTick(this.view, g.x))),
+      row: nearestTrack(this.view, asset.tracks.length, g.y),
+    };
+  }
+
+  /**
+   * The row of the picked track or of the picked clips' track, else the topmost the picked items lie
+   * on – the items just copied land on their own tracks again; -1 without.
+   */
+  private pickedTrack(asset: CompositionAsset): number {
+    const s = this.state.selection;
+    const rowOf = (track: string) => asset.tracks.findIndex((t) => t.id === track);
+    if (s?.kind === 'composition' && s.assetId === asset.id) return rowOf(s.trackId);
+    const picked = new Set(this.pickedItems(asset));
+    const rows = asset.items.filter((i) => picked.has(i.id)).map((i) => rowOf(i.track));
+    return rows.length > 0 ? Math.min(...rows) : -1;
+  }
+
   // --- test hooks --------------------------------------------------------------------------------
 
   /** Where a track and frame lie on screen, in global coordinates: the middle of the track. */
@@ -491,7 +538,8 @@ export class CompositionTimeline extends Widget {
   private onPointerMove(e: FederatedPointerEvent): void {
     const asset = this.asset();
     const local = this.toLocal(e.global);
-    const g = this.inLanes({ x: local.x, y: local.y });
+    this.pointer = { x: local.x, y: local.y };
+    const g = this.inLanes(this.pointer);
     const dragged = activeDrag();
     if (dragged?.kind === 'asset' && asset) {
       this.followDrop(asset, dragged.id, local, g);

src/player/Player.ts

@@ -1,5 +1,6 @@
 import type { SaveArea } from '../core/graph/savedValues';
 import { type Project, startScene } from '../core/model/Project';
+import { VIDEO_WAIT_MS } from '../core/model/video';
 import { gameClock, timeStep } from '../core/time';
 import { GameRun } from '../runtime/GameRun';
 import { loadPhysics, readyPhysics } from '../runtime/physics/engine';
@@ -8,7 +9,7 @@ import { Input } from '../runtime/systems/Input';
 import { downloadBlob, requestRender } from '../ui';
 import { GameRecorder } from '../viewport/GameRecorder';
 import { soundLibrary } from '../viewport/SoundLibrary';
-import { VIDEO_WAIT_MS, videoLibrary } from '../viewport/VideoLibrary';
+import { videoLibrary } from '../viewport/VideoLibrary';
 import type { GameScreen } from './GameScreen';
 
 /**
@@ -71,6 +72,7 @@ export class Player {
       clock: gameClock,
       saves: this.saves,
       recorder: this.recorder,
+      pictures: { missing: (id) => this.screen.missesVideoOf(id) },
     });
     videoLibrary.startRun();
     this.input.attach();

src/runtime/GameRun.ts

@@ -8,6 +8,7 @@ import { findScene, type Project } from '../core/model/Project';
 import { mixOf } from '../core/music/mixer';
 import type { RecordingSink } from '../core/recording/recording';
 import type { AudioSink } from './systems/Audio';
+import type { VideoPictures } from './systems/CompositionSystem';
 import { addGameSystems, type BehaviorSystem, type TimeScaled } from './systems/BehaviorSystem';
 import type { Input } from './systems/Input';
 import { bringObjects, keptEntities, objectsToBring } from './keptObjects';
@@ -21,6 +22,8 @@ export interface RunServices {
   saves: SaveArea;
   /** Outlives a `Load scene` and a restart, so a recording goes on through both (#405). */
   recorder?: RecordingSink;
+  /** The view the run is shown in, asked whether a video has its first picture yet (#417). */
+  pictures?: VideoPictures;
 }
 
 /** What a scene holds of objects, which is all a run writes into it. */

src/runtime/systems/BehaviorSystem.ts

@@ -55,7 +55,7 @@ import {
 import { CollisionSystem } from './CollisionSystem';
 import type { CollisionSource } from './solids';
 import { AnimationSystem } from './AnimationSystem';
-import { CompositionSystem } from './CompositionSystem';
+import { CompositionSystem, type VideoPictures } from './CompositionSystem';
 import type { Input } from './Input';
 import type { RecordingSink } from '../../core/recording/recording';
 import type { System } from './System';
@@ -122,6 +122,8 @@ export interface GameSystemOptions {
   recorder?: RecordingSink | null;
   /** Whether `Play composition` plays (#417); the editor run ignores it. */
   playsCompositions?: boolean;
+  /** The view a played game is shown in: a composition waits for its videos there (#417). */
+  pictures?: VideoPictures | null;
 }
 
 /** Collisions, compositions and animations are ticked first, so behaviors see this frame's events. */
@@ -139,10 +141,11 @@ export function addGameSystems(
     saves = null,
     recorder = null,
     playsCompositions = true,
+    pictures = null,
   }: GameSystemOptions = {},
 ): BehaviorSystem {
   const collisions = physics ? new PhysicsSystem(physics) : new CollisionSystem();
-  const compositions = new CompositionSystem(playsCompositions);
+  const compositions = new CompositionSystem(playsCompositions, pictures);
   const animations = new AnimationSystem();
   const sources = { collisions, animations, compositions, audio, clock, saves, recorder };
   const behaviors = new BehaviorSystem(project, input, sources, shouldRun);

src/runtime/systems/CompositionSystem.ts

@@ -13,7 +13,7 @@ import { curveLookup } from '../../core/model/curve';
 import { contentId, placedWithContent } from '../../core/model/instance';
 import { findAssetOf } from '../../core/model/Project';
 import type { PropertyValue } from '../../core/model/Properties';
-import { findVideo } from '../../core/model/video';
+import { findVideo, VIDEO_WAIT_MS } from '../../core/model/video';
 import { VIDEO_CLOCK_KEY, VIDEO_TIME_KEY, videoDrivenOf } from '../../core/properties/videos';
 import type { Entity } from '../Entity';
 import type { World } from '../World';
@@ -38,6 +38,14 @@ interface Run {
   id: number;
   /** By item id. An item a behavior destroyed stays here until it ends, so it is not built again. */
   live: Map<string, Spawned>;
+  /** Seconds the clock has stood so far for a video without its first picture, see `waits`. */
+  waited: number;
+}
+
+/** What the view that shows a run knows of its driven videos (#404). */
+export interface VideoPictures {
+  /** Whether it left the sprite `objectId` out, its video still without a first picture. */
+  missing(objectId: string): boolean;
 }
 
 /**
@@ -46,12 +54,18 @@ interface Run {
  * videos. Runs before the animations and the behaviors, so a behavior writes last and hears this
  * frame's `finished`. The editor run plays none: it writes into the saved scene, where the objects of
  * a cut would stay behind.
+ *
+ * While the view still waits for the first picture of a video an item brings in, the composition's
+ * clock stands, so the video starts at its first frame and not where the clock has got to meanwhile.
  */
 export class CompositionSystem implements System {
   finished: CompositionFinish[] = [];
   private readonly runs = new Map<Entity, Run>();
 
-  constructor(private readonly plays = true) {}
+  constructor(
+    private readonly plays = true,
+    private readonly pictures: VideoPictures | null = null,
+  ) {}
 
   update(world: World, dt: number): void {
     this.finished = [];
@@ -82,8 +96,8 @@ export class CompositionSystem implements System {
       return;
     }
     run = this.runs.get(placement);
-    if (!run) this.runs.set(placement, (run = { id: player.run, live: new Map() }));
-    if (player.shown && isHandleEnabled(placement)) player.time += dt;
+    if (!run) this.runs.set(placement, (run = { id: player.run, live: new Map(), waited: 0 }));
+    if (player.shown && isHandleEnabled(placement) && !this.waits(run, dt)) player.time += dt;
     player.shown = true;
     const length = compositionLength(world.project, comp);
     let frame = compositionFrameAt(comp, player.time);
@@ -99,6 +113,20 @@ export class CompositionSystem implements System {
     this.show(world, comp, placement, run, frame);
   }
 
+  /** Whether a video of a live item still has no picture; at most `VIDEO_WAIT_MS` at a stretch. */
+  private waits(run: Run, dt: number): boolean {
+    const pictures = this.pictures;
+    const missing =
+      pictures !== null &&
+      [...run.live.values()].some(({ videos }) => videos.some((v) => pictures.missing(v.objectId)));
+    if (!missing) {
+      run.waited = 0;
+      return false;
+    }
+    run.waited += dt;
+    return run.waited * 1000 < VIDEO_WAIT_MS;
+  }
+
   /** Takes every item of the placement's run away. */
   private end(world: World, placement: Entity): void {
     const run = this.runs.get(placement);

src/viewport/VideoLibrary.ts

@@ -40,8 +40,6 @@ const IDLE_MS = 2000;
 const SEEK_SLACK = 1 / 120;
 /** How long after `seeked` a seek counts as landed when Chrome hands over no frame for it. */
 const LAND_MS = 150;
-/** How long a run waits at its start for the driven videos it shows (#404) before it goes anyway. */
-export const VIDEO_WAIT_MS = 3000;
 
 /**
  * When the frame being drawn began, in milliseconds like `performance.now()`. The ticker moves every

src/viewport/WorldViewport.ts

@@ -125,6 +125,8 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
   private readonly videosShown = new Map<string, string>();
   /** Since when the view keeps its last frame for a driven video it brings in, see `holdsForVideo`. */
   private heldSince: number | null = null;
+  /** The sprites the last redraw left out, their driven video still without its first picture. */
+  private readonly videosLeftOut = new Set<string>();
   private cameraMoves = 0;
   private readonly stopFiles: (() => void)[];
   /** One view per overlay camera (#302), by camera id, laid over this one in camera order. */
@@ -408,6 +410,7 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
     this.content.visible = view !== null;
     this.visibleCount = 0;
     this.animated = false;
+    this.videosLeftOut.clear();
     this.drawn = null;
     if (!view || !frame) {
       this.unshadeCamera();
@@ -885,6 +888,7 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
       if (ghost) disposeSpriteMesh(ghost);
       this.ghostSprites.delete(item.id);
       this.videosShown.delete(item.id);
+      if (!videoLibrary.hasPicture(key)) this.videosLeftOut.add(item.id);
       return;
     }
     if (key !== null) this.videosShown.set(item.id, key);
@@ -927,6 +931,19 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
     return videoLibrary.loading(this.wearer);
   }
 
+  /**
+   * Whether the last redraw left out a driven video (#404) it should show, its player still without
+   * a first picture. Unlike `videosLoading`, one opened ahead or switched off does not count.
+   */
+  missesVideo(): boolean {
+    return this.videosLeftOut.size > 0;
+  }
+
+  /** Whether the last redraw left out the sprite `objectId` so, see `missesVideo`. */
+  missesVideoOf(objectId: string): boolean {
+    return this.videosLeftOut.has(objectId);
+  }
+
   /** Whether a driven video (#404) drawn here may be heard: in a run, and in the animation preview. */
   protected hearsVideo(): boolean {
     return false;