diff --git a/README.md b/README.md index ff1644b9..ec44546b 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh The browser suite and the three example games need a machine with a GPU: ```bash -node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2032 checks (GM_STEPS=20 for one step) +node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2033 checks (GM_STEPS=20 for one step) # every tool signs in first – see the Accounts page node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs diff --git a/src/core/model/composition.ts b/src/core/model/composition.ts index f9634a4b..c3d4f4ba 100644 --- a/src/core/model/composition.ts +++ b/src/core/model/composition.ts @@ -225,7 +225,39 @@ export function compositionAt(project: Project, comp: CompositionAsset, frame: n * the document. */ export function compositionHolder(project: Project, comp: CompositionAsset, frame: number): CompositionAsset { - const live = compositionAt(project, comp, frame); + return holderOf(project, comp, compositionAt(project, comp, frame)); +} + +/** How far ahead of where a composition stands the items about to begin are written out, in seconds. */ +export const AHEAD_SECONDS = 1; + +/** + * The items that begin within `seconds` after `frame` – once it loops, those at the start too when + * the end is that close – each written out as at its first frame, like `compositionHolder` does. A + * view opens their videos ahead of the cut: opened when its item began, a video had no picture yet, + * and the cut showed nothing until it loaded. One alive at `frame` is left out, it wears its own time. + */ +export function compositionAhead( + project: Project, + comp: CompositionAsset, + frame: number, + seconds = AHEAD_SECONDS, +): CompositionAsset { + const until = frame + Math.ceil(seconds * compositionFps(comp)); + const length = compositionLength(project, comp); + const wrapped = compositionLoops(comp) && until >= length ? until - length : -1; + const coming = comp.items.filter( + (item) => (item.start > frame && item.start <= until) || item.start <= wrapped, + ); + if (coming.length === 0) return holderOf(project, comp, []); + const alive = new Set(compositionAt(project, comp, frame).map((l) => l.item.id)); + const live = coming + .filter((item) => !alive.has(item.id)) + .flatMap((item) => compositionAt(project, comp, item.start).filter((l) => l.item.id === item.id)); + return holderOf(project, comp, live); +} + +function holderOf(project: Project, comp: CompositionAsset, live: LiveItem[]): CompositionAsset { const holder: CompositionAsset = { ...comp, props: { ...comp.props }, diff --git a/src/editor/composition/CompositionCanvas.ts b/src/editor/composition/CompositionCanvas.ts index c77d223f..3d8d49b1 100644 --- a/src/editor/composition/CompositionCanvas.ts +++ b/src/editor/composition/CompositionCanvas.ts @@ -1,6 +1,8 @@ import { compositionHolder } from '../../core/model/composition'; import type { CompositionAsset } from '../../core/model/Asset'; import { findAssetOf } from '../../core/model/Project'; +import { compositionVideosAhead } from '../../viewport/items'; +import type { Frame } from '../../viewport/WorldViewport'; import type { EditorState } from '../state'; import { SceneViewport } from '../viewport/SceneViewport'; @@ -45,7 +47,7 @@ export class CompositionCanvas extends SceneViewport { shownHolder(): CompositionAsset | undefined { const comp = this.assetId ? findAssetOf(this.state.project, this.assetId, 'composition') : undefined; if (!comp) return undefined; - const frame = Math.max(0, Math.floor(this.playhead() + 1e-6)); + const frame = this.frameAt(); if (this.shown?.frame !== frame || this.shown.version !== this.version) this.shown = { frame, @@ -59,6 +61,19 @@ export class CompositionCanvas extends SceneViewport { return this.shownHolder(); } + /** With the videos of the items about to begin worn ahead, so a cut to one shows its picture at once. */ + protected override collectFrame(): Frame { + const frame = super.collectFrame(); + const comp = this.assetId ? findAssetOf(this.state.project, this.assetId, 'composition') : undefined; + if (!comp) return frame; + return { ...frame, ahead: compositionVideosAhead(this.state.project, comp, this.frameAt()) }; + } + + /** The frame the playhead stands on. */ + private frameAt(): number { + return Math.max(0, Math.floor(this.playhead() + 1e-6)); + } + /** Each composition keeps the view it was left with, as a container does (#171). */ protected override framesOnOpen(): boolean { return false; diff --git a/src/editor/debug/documentHooks.ts b/src/editor/debug/documentHooks.ts index 83c03c7b..00f7d7f2 100644 --- a/src/editor/debug/documentHooks.ts +++ b/src/editor/debug/documentHooks.ts @@ -92,6 +92,8 @@ export function documentHooks({ state, editor }: HookContext) { shown: () => (editor.compositionEditor.shownHolder()?.objects ?? []).map((o) => ({ id: o.id, x: o.position.x })), objectScreen: (objectId: string) => editor.compositionEditor.preview.objectScreen(objectId), + /** Where the picture draws a sprite right now; null while it is left out, a video without its picture. */ + drawn: (objectId: string) => editor.compositionEditor.preview.drawnRect(objectId), }, tile: { current: () => editor.tileEditor.currentAssetId, diff --git a/src/viewport/WorldViewport.ts b/src/viewport/WorldViewport.ts index 2bdfd373..a1fa260b 100644 --- a/src/viewport/WorldViewport.ts +++ b/src/viewport/WorldViewport.ts @@ -17,7 +17,7 @@ import type { Project } from '../core/model/Project'; import { cancelFrameWork, onFrame, type Size, ui, Widget } from '../ui'; import { fontLibrary } from './FontLibrary'; import { drawSlots } from './drawOrder'; -import type { DrawItem } from './items'; +import type { DrawItem, VideoAhead } from './items'; import { hashId } from '../core/particles/nodes'; import { ParticleField } from './particles'; import { particleLibrary } from './ParticleLibrary'; @@ -57,6 +57,8 @@ export interface ViewSource { export interface Frame { items: DrawItem[]; background: number; + /** Driven videos (#404) worn without being drawn: those of composition items about to begin (#417). */ + ahead?: VideoAhead[]; } /** A part of the view by its centre and size, in local CSS pixels. */ @@ -465,6 +467,8 @@ export abstract class WorldViewport extends W this.isSelected(item.id), ); } + // Opened before their item begins, so the cut to it has its picture at once. + for (const ahead of frame.ahead ?? []) if (this.wearVideo(ahead, false) !== ahead) this.animated = true; this.afterItems(items, view); this.drawOnScreen(items, view); this.drawn = { scale: view.scale, w: rect.w, h: rect.h, grid: view.grid ?? null, culled }; @@ -903,7 +907,7 @@ export abstract class WorldViewport extends W * The item with the player of its own a driven video (#404) has, per view and object. One that is * not `seen` is never heard. */ - private wearVideo(item: DrawItem, seen = true): DrawItem { + private wearVideo(item: T, seen = true): T { if (!item.texture) return item; const texture = videoLibrary.wear( item.texture, @@ -914,7 +918,7 @@ export abstract class WorldViewport extends W return texture === item.texture ? item : { ...item, texture }; } - private videoWearer(item: DrawItem): string { + private videoWearer(item: Pick): string { return `${this.wearer}:${item.id}`; } diff --git a/src/viewport/items.ts b/src/viewport/items.ts index 287f19bd..5b8df838 100644 --- a/src/viewport/items.ts +++ b/src/viewport/items.ts @@ -1,11 +1,11 @@ import { isHandleEnabled, worldTransformOf } from '../core/graph'; import { type AssetType, isSymbolType } from '../core/model/AssetTypes'; import { treeOrder } from '../core/model/hierarchy'; -import { placedPicture } from '../core/model/composition'; -import { resolvedHolder } from '../core/model/instance'; +import { compositionAhead, compositionFrameAt, placedPicture } from '../core/model/composition'; +import { contentId, placedWithContent, resolvedHolder } from '../core/model/instance'; import type { Mat } from '../core/model/matrix'; import type { PropertyBag } from '../core/model/Properties'; -import type { ParticlesAsset, TilemapAsset } from '../core/model/Asset'; +import type { CompositionAsset, ParticlesAsset, TilemapAsset } from '../core/model/Asset'; import { readTile, tileFps, @@ -25,11 +25,12 @@ import { triggerRects, } from '../core/model/tilemapCollision'; import { findAsset, findAssetOf, type ObjectHolder, type Project } from '../core/model/Project'; -import { objectName } from '../core/model/SceneObject'; +import { objectName, type SceneObject } from '../core/model/SceneObject'; import { fontFileOf, fontSmoothOf } from '../core/model/font'; import { resolveTexture, type TextureRef } from '../core/model/texture'; import type { FrameOutline } from '../core/model/frameOutline'; import { liveTilemap, type TileEdits } from '../core/model/liveTilemap'; +import { isVideoKey } from '../core/model/video'; import { renderLayerAlong } from '../core/model/renderLayers'; import { type ClipRegion, clipRegions } from '../core/properties/clip'; import { colliderFrame } from '../core/properties/collider'; @@ -84,9 +85,53 @@ export interface DrawItem { sealedIn?: string; } +/** A sprite not drawn yet whose driven video a view opens ahead (#417), under the id it will be drawn by. */ +export type VideoAhead = Pick; + interface DrawList { items: DrawItem[]; background: number; + /** The videos of composition items about to begin, see `compositionAhead`. */ + ahead?: VideoAhead[]; +} + +/** + * The sprites among `objects` – already laid out, container content and all – whose video is driven: + * what a view wears ahead for a composition's coming items. + */ +function drivenVideos(project: Project, objects: readonly SceneObject[]): VideoAhead[] { + return objects.flatMap((object) => { + const asset = findAsset(project, object.assetId); + if (asset?.type !== 'sprite') return []; + const props = resolveSceneObjectProps(project, asset, object); + const texture = resolveTexture(project, props); + return texture && isVideoKey(texture.file) && videoDrivenOf(props) + ? [{ id: object.id, texture, props }] + : []; + }); +} + +/** The videos of a composition's items about to begin after `frame`, by the ids its own tab draws them by. */ +export function compositionVideosAhead( + project: Project, + comp: CompositionAsset, + frame: number, +): VideoAhead[] { + return drivenVideos(project, resolvedHolder(project, compositionAhead(project, comp, frame)).objects); +} + +/** The same for every composition the run plays, by the ids its items take below their placement. */ +function worldVideosAhead(world: World): VideoAhead[] { + return world.entities.flatMap((placement) => { + const player = placement.composition; + const comp = player && !player.stopped && findAssetOf(world.project, placement.assetId, 'composition'); + if (!comp) return []; + const frame = compositionFrameAt(comp, player.time); + const objects = compositionAhead(world.project, comp, frame).objects.flatMap((o) => + placedWithContent(world.project, o, contentId(placement.objectId, o.id), placement.objectId), + ); + return drivenVideos(world.project, objects); + }); } /** One tile a tilemap paints with, as drawing needs it: the blocks and how they run. */ @@ -340,5 +385,5 @@ export function collectWorldItems(world: World): DrawList { (at) => at.parent ?? null, ), })); - return { items, background: world.backgroundColor }; + return { items, background: world.backgroundColor, ahead: worldVideosAhead(world) }; } diff --git a/tests/core/composition-play.test.ts b/tests/core/composition-play.test.ts index faa70bab..c5fd3694 100644 --- a/tests/core/composition-play.test.ts +++ b/tests/core/composition-play.test.ts @@ -24,7 +24,7 @@ import { addGameSystems } from '../../src/runtime/systems/BehaviorSystem'; import { Input } from '../../src/runtime/systems/Input'; import { World } from '../../src/runtime/World'; import { stackAt } from '../../src/viewport/pick'; -import { collectHolderItems } from '../../src/viewport/items'; +import { collectHolderItems, collectWorldItems } from '../../src/viewport/items'; import { graphEditor, makeBehavior, makeProject, type ProjectFixture } from '../helpers'; import { PlaceAssetCommand } from '../../src/core/commands/sceneObjectCommands'; @@ -224,6 +224,36 @@ describe('a composition played in a scene', () => { expect(run.at(item)!.props[VIDEO_TIME_KEY]).toBeCloseTo(2, 9); }); + it('hands the view the video of the next item a second ahead, by the id the item will take', () => { + const stage = makeStage(); + director(stage); + const video = new CreateVideoCommand('Intro', FILE, { + width: 320, + height: 180, + duration: 10, + fps: 30, + audio: false, + }); + stage.stack.execute(video); + const screen = new CreateAssetCommand('sprite', 'Screen', { texture: video.asset.id }); + stage.stack.execute(screen); + stage.item(first(stage), screen.asset.id, 0, 20); + const next = stage.item(first(stage), screen.asset.id, 20, 10, 0.5); + const run = play(stage); + const ahead = () => collectWorldItems(run.world).ahead ?? []; + run.frame(10); + expect(ahead()).toEqual([]); + run.frame(); + const [coming] = ahead(); + expect(coming?.id).toBe(contentId(stage.placement, next)); + expect(coming?.texture?.file).toMatch(/^video:/); + expect(coming?.props[VIDEO_CLOCK_KEY]).toBe('driven'); + expect(coming?.props[VIDEO_TIME_KEY]).toBe(0.5); + run.frame(10); + expect(run.at(next)).toBeDefined(); + expect(ahead()).toEqual([]); + }); + it('loops without finishing, building an item anew each time round', () => { const stage = makeStage(); director(stage); diff --git a/tests/core/composition.test.ts b/tests/core/composition.test.ts index fcfae15a..39adfed1 100644 --- a/tests/core/composition.test.ts +++ b/tests/core/composition.test.ts @@ -24,6 +24,7 @@ import type { } from '../../src/core/model/Asset'; import { clipLength, + compositionAhead, compositionAt, compositionFps, compositionHolder, @@ -35,6 +36,7 @@ import { import { instanceChildId, resolveObjects } from '../../src/core/model/instance'; import { findAssetOf, objectHolders } from '../../src/core/model/Project'; import type { PropertyBag } from '../../src/core/model/Properties'; +import { COMPOSITION_LOOP_KEY } from '../../src/core/properties/compositions'; import { createSceneObject } from '../../src/core/model/SceneObject'; import { VIDEO_CLOCK_KEY, @@ -532,6 +534,38 @@ describe('items wearing a video', () => { ).toBe(180); }); + it('are written out a second before they begin, at their first frame, so a view opens them ahead', () => { + const cut = makeCut(); + const screen = videoSprite(cut, addVideo(cut)); + const track = firstTrack(cut); + cut.item(track, screen, 0, 120); + const next = cut.item(track, screen, 120, 120, { offset: 2 }); + const ahead = (frame: number) => compositionAhead(cut.project, cut.comp(), frame).objects; + expect(ahead(59)).toEqual([]); + const [object] = ahead(60); + expect(object!.id).toBe(next); + expect(object!.overrides[VIDEO_CLOCK_KEY]).toBe('driven'); + expect(object!.overrides[VIDEO_TIME_KEY]).toBe(2); + // Begun, it wears its own time and is no longer ahead. + expect(ahead(120)).toEqual([]); + }); + + it('at the start are ahead too when it loops and its end comes close – unless still alive', () => { + const cut = makeCut(); + const screen = videoSprite(cut, addVideo(cut)); + const track = firstTrack(cut); + const first = cut.item(track, screen, 0, 120); + cut.item(track, screen, 120, 120); + const ids = (frame: number) => compositionAhead(cut.project, cut.comp(), frame).objects.map((o) => o.id); + expect(ids(200)).toEqual([]); + cut.comp().props[COMPOSITION_LOOP_KEY] = true; + expect(ids(179)).toEqual([]); + expect(ids(200)).toEqual([first]); + cut.comp().items[0]!.length = 240; + cut.comp().items.pop(); + expect(ids(200)).toEqual([]); + }); + it('follow the texture an override gives the item', () => { const cut = makeCut(); const item = cut.item(firstTrack(cut), cut.spriteId, 0, 1); diff --git a/tools/e2e/69-composition-tab.mjs b/tools/e2e/69-composition-tab.mjs index 9ff8a536..6388c329 100644 --- a/tools/e2e/69-composition-tab.mjs +++ b/tools/e2e/69-composition-tab.mjs @@ -3,13 +3,19 @@ * and picked, so the inspector shows its TIMING; a second one dropped onto the first lands right * after it. A dragged end stops at the next item. An animation dragged in becomes a clip that moves * the item it overlaps in the picture. The hierarchy lists what is alive at the playhead, Delete - * takes an item with its object, and Play runs the playhead to the end. + * takes an item with its object, and Play runs the playhead to the end. The video of an item about + * to begin is opened ahead, so the cut to it shows its picture at once. * * Everything it adds carries a `ct69` id and goes again at the end. */ +import { readFileSync } from 'node:fs'; +import { uploadVideo } from '../lib/projects.mjs'; + const CUT = 'ct69-cut'; const BOX = 'ct69-box'; const SLIDE = 'ct69-slide'; +const VIDEO = 'ct69-video'; +const SCREEN = 'ct69-screen'; export async function compositionTab(t) { const { check, sleep, json, evaluate, mutate, mouse, drag, press, showTab } = t; @@ -182,6 +188,48 @@ export async function compositionTab(t) { JSON.stringify({ running, ended }), ); + // 69h. Two video items back to back: the second one's video is opened a second ahead, so the cut + // draws its picture in the very redraw it begins – loaded only then, it was left out until it came. + const uploaded = await uploadVideo( + t.baseUrl, + readFileSync(new URL('./fixtures/colors.webm', import.meta.url)), + 'colors.webm', + ); + await mutate(` + const p = __gm.state.project; + p.assets.push( + { id: '${VIDEO}', type: 'video', name: 'Colors 69', props: {}, file: ${JSON.stringify(uploaded.file)}, + width: 320, height: 180, duration: 2, fps: 30, audio: true }, + { id: '${SCREEN}', type: 'sprite', name: 'Screen 69', props: { width: 160, height: 90, color: 0xffffff, texture: '${VIDEO}' } }, + ); + const c = p.assets.find((a) => a.id === '${CUT}'); + const object = (id) => ({ ...structuredClone(c.objects[0]), id, assetId: '${SCREEN}', overrides: {}, instanceEdits: {} }); + c.objects = [object('ct69-v1'), object('ct69-v2')]; + c.items = [ + { id: 'ct69-v1', track: 'ct69-t1', start: 0, length: 20, offset: 0 }, + { id: 'ct69-v2', track: 'ct69-t1', start: 20, length: 10, offset: 1 }, + ]; + c.clips = [];`); + const second = `__gm.videos().find((v) => v.key.endsWith(':ct69-v2')) ?? null`; + await evaluate(`__gm.composition.scrub(5)`); + await sleep(300); + const early = await json(second); + await evaluate(`__gm.composition.scrub(12)`); + let ahead = null; + // Settled: a seek still landing would hold the view a frame, which is no picture missing. + for (let i = 0; i < 40 && !(ahead?.ready && !ahead.seeking && !ahead.landing); i++) { + await sleep(150); + ahead = await json(second); + } + const cut = await json( + `(() => { __gm.composition.scrub(20); return __gm.composition.drawn('ct69-v2'); })()`, + ); + check( + 'a coming video item is opened a second ahead, at its offset, and drawn in the redraw it begins', + early === null && ahead?.ready && ahead.paused && Math.abs(ahead.time - 1) < 0.05 && cut !== null, + JSON.stringify({ early, ahead, cut }), + ); + await showTab('scene'); await mutate(` const p = __gm.state.project;