diff --git a/README.md b/README.md index 1ce9725a..28c7d670 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh The browser suite and the three example games need a machine with a GPU: ```bash -node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2074 checks (GM_STEPS=20 for one step) +node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2077 checks (GM_STEPS=20 for one step) # every tool signs in first – see the Accounts page node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs diff --git a/src/core/model/Asset.ts b/src/core/model/Asset.ts index b959362a..39a2f4ca 100644 --- a/src/core/model/Asset.ts +++ b/src/core/model/Asset.ts @@ -305,6 +305,7 @@ export interface CompositionAudio { kind: CompositionAudioKind; asset: string; volume: number; + /** '' along the routing of its asset, `TO_COMPOSITION_BUS` into the composition's bus, else as a Play node picks. */ bus: string; } diff --git a/src/core/model/texture.ts b/src/core/model/texture.ts index 02cf36b0..8b232c42 100644 --- a/src/core/model/texture.ts +++ b/src/core/model/texture.ts @@ -4,7 +4,17 @@ import { findPixelFrame, pixelFrameRect } from './pixelart'; import { type PropertyBag, textOf, type Vec2 } from './Properties'; import { findAsset, type Project } from './Project'; import { storedFilePattern } from './storedFile'; -import { findVideo, isVideoFile, videoKey, videoPlayOf, type VideoRef } from './video'; +import { videoBusOf } from '../properties/videos'; +import { + findVideo, + isVideoFile, + isVideoKey, + routedVideoKey, + videoKey, + videoMixBus, + videoPlayOf, + type VideoRef, +} from './video'; const FILE_PATTERN = storedFilePattern('png|jpg|webp|gif'); @@ -93,7 +103,10 @@ export function textureRefIds(asset: TextureAsset): string[] { /** What a sprite wearing `props.texture` shows. */ export function resolveTexture(project: Project, props: PropertyBag): TextureRef | null { - return resolvePicture(project, textOf(props.texture ?? '')); + const ref = resolvePicture(project, textOf(props.texture ?? '')); + // A video heard on a bus the sprite picked plays in a player of its own (#418). + const bus = ref && isVideoKey(ref.file) ? videoMixBus(project, videoBusOf(props)) : null; + return bus ? { ...ref!, file: routedVideoKey(ref!.file, bus) } : ref; } /** A texture, a frame, a video or a clip of one, or a generating asset; any other asset's id is no picture. */ diff --git a/src/core/model/video.ts b/src/core/model/video.ts index a57ffad5..48954696 100644 --- a/src/core/model/video.ts +++ b/src/core/model/video.ts @@ -1,6 +1,7 @@ +import { FROM_ASSET_BUS, MASTER_MIX, mixBusName, pickedComposition } from '../properties/buses'; import { videoLoopsOf, videoSpanOf, videoVolumeOf } from '../properties/videos'; import type { VideoAsset, VideoClip } from './Asset'; -import { findAsset, type Project } from './Project'; +import { findAsset, findAssetOf, type Project } from './Project'; import { clipsOf } from './sound'; import { storedFilePattern } from './storedFile'; @@ -64,6 +65,8 @@ export interface VideoPlay { end: number; loop: boolean; volume: number; + /** The bus of the mix its sound goes into, where a sprite picked one; along its routing where left out. */ + bus?: string; } export function videoPlayOf(ref: VideoRef): VideoPlay { @@ -107,12 +110,33 @@ const PREFIX = 'video:'; /** * The name a playing video goes by where a picture is named by `TextureRef.file` (see - * `generated.ts`): `video:#~~~`. No uploaded image looks like it, - * and one key is one player: two sprites wearing the same clip show the same picture. + * `generated.ts`): `video:#~~~`, and `!` where its sound goes + * into a bus a sprite picked. No uploaded image looks like it, and one key is one player: two sprites + * wearing the same clip show the same picture. */ export function videoKey(play: VideoPlay): string { const end = Number.isFinite(play.end) ? play.end : 0; - return `${PREFIX}${play.file}#${play.start}~${end}~${play.loop ? 1 : 0}~${play.volume}`; + const bus = play.bus ? `!${play.bus}` : ''; + return `${PREFIX}${play.file}#${play.start}~${end}~${play.loop ? 1 : 0}~${play.volume}${bus}`; +} + +/** + * The bus of the mix the Video bus of a sprite names: null along the routing of its video, the + * composition's for one it picked for a sprite of its items. A bus that is gone is the master. + */ +export function videoMixBus(project: Project, picked: string): string | null { + if (picked === '' || picked === FROM_ASSET_BUS) return null; + const comp = pickedComposition(picked); + if (comp !== null) return findAssetOf(project, comp, 'composition') ? mixBusName(comp) : null; + return findAssetOf(project, picked, 'bus') ? mixBusName(picked) : MASTER_MIX; +} + +/** The key of the same video heard on `bus` (#418); along its routing for null. */ +export function routedVideoKey(key: string, bus: string | null): string { + const play = parseVideoKey(key); + if (!play || isDrivenVideoKey(key)) return key; + const { bus: _, ...rest } = play; + return videoKey(bus ? { ...rest, bus } : rest); } /** @@ -138,11 +162,14 @@ export function parseVideoKey(key: string): VideoPlay | null { if (hash < 0) return null; const file = key.slice(PREFIX.length, hash); const at = key.indexOf('@', hash); - const parts = key.slice(hash + 1, at < 0 ? undefined : at).split('~'); + const body = key.slice(hash + 1, at < 0 ? undefined : at); + const bang = body.indexOf('!'); + const parts = (bang < 0 ? body : body.slice(0, bang)).split('~'); const [start, end, loop, volume] = parts.map(Number); if (!isVideoFile(file) || parts.length !== 4 || ![start, end, loop, volume].every(Number.isFinite)) return null; - return { file, start: start!, end: end! > 0 ? end! : Infinity, loop: loop === 1, volume: volume! }; + const play = { file, start: start!, end: end! > 0 ? end! : Infinity, loop: loop === 1, volume: volume! }; + return bang < 0 ? play : { ...play, bus: body.slice(bang + 1) }; } /** How far before its end a held clip shows its last picture: the end itself is the next one's. */ diff --git a/src/core/music/compositionAudio.ts b/src/core/music/compositionAudio.ts index b6c79559..6211857d 100644 --- a/src/core/music/compositionAudio.ts +++ b/src/core/music/compositionAudio.ts @@ -13,7 +13,7 @@ import { compositionFps, compositionLength, compositionLoops, itemFades } from ' import { findAssetOf, type Project } from '../model/Project'; import { findSound } from '../model/sound'; import { pickedFx } from '../properties/assetFx'; -import { MASTER_MIX, type MixRoute, playedRoute } from './mixer'; +import { itemRoute, MASTER_MIX, type MixRoute } from './mixer'; import { fxVoiceOf } from './needs'; import { patternInstrument, patternLength, patternSequence, patternTempo } from './pattern'; import type { BusLevel } from './playbacks'; @@ -67,7 +67,7 @@ export function compositionPlayings( if (!item.audio || !tracks.has(item.track)) continue; const laid = laidOut(project, comp.id, item, fps); if (!laid) continue; - const route = routed ? playedRoute(project, item.audio.bus, laid.routedAs) : MASTER_MIX; + const route = routed ? itemRoute(project, comp.id, item.audio.bus, laid.routedAs) : MASTER_MIX; const key = JSON.stringify(route); let playing = byRoute.get(key); if (!playing) diff --git a/src/core/music/mixer.ts b/src/core/music/mixer.ts index f2269467..4da53408 100644 --- a/src/core/music/mixer.ts +++ b/src/core/music/mixer.ts @@ -1,6 +1,13 @@ import type { SendTap } from '../graph/Graph'; import { findAssetOf, type Project } from '../model/Project'; -import { BUS_FX_KEY, busVolumeOf, MASTER_BUS } from '../properties/buses'; +import { + BUS_FX_KEY, + busVolumeOf, + MASTER_BUS, + MASTER_MIX, + mixBusName, + TO_COMPOSITION_BUS, +} from '../properties/buses'; import { coerceAs } from '../properties/kinds'; import type { FxVoice } from '../soundfx/nodes'; import { fxVoiceOf } from './needs'; @@ -9,8 +16,7 @@ import { sendOf } from './routing'; import { dbToGain } from './song'; import { DEFAULT_TEMPO } from './time'; -/** What the sound library calls the master bus (#325); every other bus of the mix is `mix:`. */ -export const MASTER_MIX = 'mix'; +export { MASTER_MIX, mixBusName }; /** One link of the master routing (#351) as the mix plays it: into that bus, at that level, from that tap. */ export interface MixSend { @@ -36,11 +42,6 @@ export interface MixBus { fx: FxVoice | null; } -/** The name of a bus of the mix by the id of its asset; the master for '' and for `MASTER_BUS`. */ -export function mixBusName(busId: string): string { - return busId === '' || busId === MASTER_BUS ? MASTER_MIX : `mix:${busId}`; -} - /** * Where an asset goes by the master routing (#351). One the routing has no node for goes into the * master, as every asset of a project written before there was a routing did. A link to what is @@ -62,9 +63,10 @@ export function routeOf(project: Project, assetId: string): MixRoute { const OWN_BUS_TYPES: readonly string[] = ['sound', 'instrument', 'video']; /** - * The master and every bus of the project (#325), each where the routing sends it (#351); and a bus - * of its own for each sound, instrument and video the routing sends anywhere but straight into one - * bus, for all its voices to go through. A sound FX that does not compile is none. + * The master and every bus of the project (#325), each where the routing sends it (#351); a bus of + * its own for each sound, instrument and video the routing sends anywhere but straight into one bus, + * for all its voices to go through; and one for each composition, for its items. A sound FX that + * does not compile is none. */ export function mixOf(project: Project): MixBus[] { const fxOf = (id: string) => fxVoiceOf(project, id, DEFAULT_TEMPO); @@ -80,6 +82,9 @@ export function mixOf(project: Project): MixBus[] { fx: fxOf(coerceAs(asset.props[BUS_FX_KEY], 'asset', '')), }, ]; + // A composition's bus is where its items go (#418), whatever its routing. + if (asset.type === 'composition') + return [{ name: mixBusName(asset.id), volume: 1, ...routed(routeOf(project, asset.id)), fx: null }]; if (!OWN_BUS_TYPES.includes(asset.type) || !isRouted(asset)) return []; const route = routeOf(project, asset.id); return typeof route === 'string' @@ -146,6 +151,14 @@ export function playedRoute(project: Project, picked: string, assetId: string): return routeOf(project, assetId); } +/** + * Where an audio item of a composition goes: into the composition's bus for `TO_COMPOSITION_BUS`, + * else as a node's pick (`playedRoute`) – along the routing of what it plays where it picks none. + */ +export function itemRoute(project: Project, compId: string, picked: string, assetId: string): MixRoute { + return picked === TO_COMPOSITION_BUS ? mixBusName(compId) : playedRoute(project, picked, assetId); +} + /** The bus of the mix a node reaches where an empty pick is the master: Set bus volume, Bus volume. */ export function reachedBus(project: Project, picked: string): string | null { if (picked === '' || picked === MASTER_BUS) return MASTER_MIX; diff --git a/src/core/music/projectRouting.ts b/src/core/music/projectRouting.ts index 8771dc20..bfc1cc5a 100644 --- a/src/core/music/projectRouting.ts +++ b/src/core/music/projectRouting.ts @@ -1,6 +1,6 @@ /** * The master routing of a project (#351): a graph with a node per sound, instrument, pattern, song, - * video with sound and bus, and the master, its links the sends from one to the next. Its nodes follow the assets: one + * video with sound, composition and bus, and the master, its links the sends from one to the next. Its nodes follow the assets: one * that comes gets a node of its own and the links most others of its type have, one that goes takes * its node along. */ @@ -16,7 +16,15 @@ import { isEmbedded } from './embedded'; import { addLink, heldSend } from './routing'; import { ROUTED_ASSET, ROUTED_KIND, ROUTED_MASTER, ROUTED_NAME } from './routingNodes'; -const ROUTED_TYPES: readonly AssetType[] = ['sound', 'instrument', 'pattern', 'song', 'video', 'bus']; +const ROUTED_TYPES: readonly AssetType[] = [ + 'sound', + 'instrument', + 'pattern', + 'song', + 'video', + 'composition', + 'bus', +]; /** What has its own sound FX and level on a bus of its own, and so may send from before them. */ const TAPPED_TYPES: readonly AssetType[] = ['bus', 'song']; @@ -133,11 +141,11 @@ export function heldRouting(graph: Graph | undefined, assets: readonly Asset[], } /** - * The links a new sound, instrument, pattern, song or video takes, so it is heard where the others are: those - * of the one whose targets most others of its type share too – the latest of them on a tie –, or of - * all that plays where nothing of its type is linked yet. Videos and the music go by each other - * apart: a video where no other is linked goes to the master, not where the sounds go. Empty where - * nothing is, and for a bus. + * The links a new sound, instrument, pattern, song, video or composition takes, so it is heard where + * the others are: those of the one whose targets most others of its type share too – the latest of + * them on a tie –, or of all that plays where nothing of its type is linked yet. Videos, compositions + * and the music go by each other apart: a video where no other is linked goes to the master, not + * where the sounds go, and so does a composition, a mix of its own. Empty where nothing is, and for a bus. */ function linksLikeTheirs( kind: string, @@ -162,12 +170,17 @@ function linksLikeTheirs( }; const playing = nodes.filter((node) => { const of = routedKind(node); - return of !== '' && of !== 'bus' && (of === 'video') === (kind === 'video') && out.has(node.id); + return of !== '' && of !== 'bus' && peersOf(of) === peersOf(kind) && out.has(node.id); }); const own = playing.filter((node) => routedKind(node) === kind); return pick(own.length > 0 ? own : playing); } +/** Whose links a new node may take: those of its own kind, else of what plays alike. */ +function peersOf(kind: string): string { + return kind === 'video' || kind === 'composition' ? kind : 'music'; +} + /** A link like another, out of a new node: as loud, and after the sound FX where it cannot tap before. */ function copiedLink(edge: GraphEdge, from: string, kind: string): GraphEdge { const raw = heldSend(edge.send); diff --git a/src/core/music/routingNodes.ts b/src/core/music/routingNodes.ts index 1538e56f..6fa99109 100644 --- a/src/core/music/routingNodes.ts +++ b/src/core/music/routingNodes.ts @@ -41,8 +41,8 @@ export function isSongRoutingDef(def: NodeDef): boolean { } /** - * The nodes of the master routing (#351): one per sound, instrument, pattern, song, video with sound - * and bus of the project, named and typed after its asset, and the master. Only a bus and the master + * The nodes of the master routing (#351): one per sound, instrument, pattern, song, video with sound, + * composition and bus of the project, named and typed after its asset, and the master. Only a bus and the master * take links in. */ export const ROUTED_ASSET = 'RoutedAsset'; @@ -56,6 +56,7 @@ const KIND_WORDS: Record = { pattern: 'Pattern', song: 'Song', video: 'Video', + composition: 'Composition', bus: 'Bus', }; @@ -64,7 +65,7 @@ registerNode({ label: 'Asset', category: 'sound', description: - 'A sound, an instrument, a pattern, a song, a video or a bus of the project: all it plays, sent on to every link.', + 'A sound, an instrument, a pattern, a song, a video, a composition or a bus of the project: all it plays, sent on to every link.', inputs: [], outputs: [], params: [], diff --git a/src/core/properties/PropertySchema.ts b/src/core/properties/PropertySchema.ts index 5e93b278..a982f743 100644 --- a/src/core/properties/PropertySchema.ts +++ b/src/core/properties/PropertySchema.ts @@ -153,6 +153,10 @@ type Extras = K extends 'number' assetTypes?: readonly AssetType[]; none?: string; master?: boolean; + /** Offers `FROM_ASSET_BUS`, where empty means a composition's bus. */ + fromAsset?: boolean; + /** Offers `TO_COMPOSITION_BUS`, where empty means the routing of the asset. */ + toComposition?: boolean; noFx?: boolean; } : K extends 'texture' diff --git a/src/core/properties/assetRef.ts b/src/core/properties/assetRef.ts index eb8b25a8..84e16d47 100644 --- a/src/core/properties/assetRef.ts +++ b/src/core/properties/assetRef.ts @@ -4,7 +4,13 @@ import { findAsset, type Project } from '../model/Project'; import { clipsOf, findSound, soundRefName } from '../model/sound'; import { listedAssets, markedName } from '../music/embedded'; import { NO_FX } from './assetFx'; -import { MASTER_BUS } from './buses'; +import { + FROM_ASSET_BUS, + FROM_ASSET_LABEL, + MASTER_BUS, + TO_COMPOSITION_BUS, + TO_COMPOSITION_LABEL, +} from './buses'; export const NO_ASSET_LABEL = 'None'; export const MASTER_LABEL = 'Master'; @@ -17,6 +23,8 @@ export const MASTER_LABEL = 'Master'; export function assetRefLabel(project: Project | null, id: string): string { if (id === '' || id === NO_FX) return NO_ASSET_LABEL; if (id === MASTER_BUS) return MASTER_LABEL; + if (id === FROM_ASSET_BUS) return FROM_ASSET_LABEL; + if (id === TO_COMPOSITION_BUS) return TO_COMPOSITION_LABEL; const asset = project ? findAsset(project, id) : undefined; if (asset) return markedName(asset); // A clip of a sound (#357) reads "Sound / Clip", as a frame does. diff --git a/src/core/properties/buses.ts b/src/core/properties/buses.ts index b00364c1..5beb7d22 100644 --- a/src/core/properties/buses.ts +++ b/src/core/properties/buses.ts @@ -13,6 +13,39 @@ export const BUS_FX_KEY = 'fx'; */ export const MASTER_BUS = '#master'; +/** What the sound library calls the master bus (#325); every other bus of the mix is `mix:`. */ +export const MASTER_MIX = 'mix'; + +/** The name of a bus of the mix by the id of its asset; the master for '' and for `MASTER_BUS`. */ +export function mixBusName(busId: string): string { + return busId === '' || busId === MASTER_BUS ? MASTER_MIX : `mix:${busId}`; +} + +/** + * What a video sprite of a composition picks to go along the routing of its video, where empty sends + * it into the composition's own bus (#418). + */ +export const FROM_ASSET_BUS = '#asset'; +export const FROM_ASSET_LABEL = 'From asset'; +/** What an audio item picks to go into its composition's own bus, where empty goes along the routing of its asset. */ +export const TO_COMPOSITION_BUS = '#composition'; +export const TO_COMPOSITION_LABEL = 'To composition'; + +const COMPOSITION_BUS = '#composition:'; + +/** + * The Video bus a composition hands a sprite of its items that picked none, as it spawns them in a + * run: that composition's bus. Never saved. + */ +export function compositionBusPick(compId: string): string { + return COMPOSITION_BUS + compId; +} + +/** The composition a Video bus names by `compositionBusPick`; null for any other. */ +export function pickedComposition(picked: string): string | null { + return picked.startsWith(COMPOSITION_BUS) ? picked.slice(COMPOSITION_BUS.length) : null; +} + export function busVolumeOf(bag: PropertyBag): number { return Math.max(0, Math.min(1, coerceAs(bag[BUS_VOLUME_KEY], 'number', 1))); } diff --git a/src/core/properties/videos.ts b/src/core/properties/videos.ts index 468e667d..d5f73ff5 100644 --- a/src/core/properties/videos.ts +++ b/src/core/properties/videos.ts @@ -1,4 +1,5 @@ import { booleanOf, type PropertyBag } from '../model/Properties'; +import { FROM_ASSET_LABEL } from './buses'; import { coerceAs } from './kinds'; import type { PropertyDef } from './PropertySchema'; @@ -23,6 +24,11 @@ export const VIDEO_CLOCK_KEY = 'videoClock'; export const VIDEO_TIME_KEY = 'videoTime'; /** How loud a driven video is heard through this sprite, 0–1, on top of the video's own Volume; a composition's fades set it. */ export const VIDEO_LEVEL_KEY = 'videoVolume'; +/** + * The bus of the mix the video's sound goes into through this sprite: empty along the routing of the + * video, in a composition into its bus; `MASTER_BUS`, a bus, or `FROM_ASSET_BUS` for the routing there. + */ +export const VIDEO_BUS_KEY = 'videoBus'; export const VIDEO_CLOCK_OPTIONS = [ { value: 'free', label: 'Free' }, @@ -60,6 +66,16 @@ export const SPRITE_VIDEO_ROWS: readonly PropertyDef[] = [ shownFor: { key: VIDEO_CLOCK_KEY, values: ['driven'] }, shownForVideo: 'texture', }, + { + key: VIDEO_BUS_KEY, + label: 'Video bus', + kind: 'asset', + assetTypes: ['bus'], + none: FROM_ASSET_LABEL, + master: true, + default: '', + shownForVideo: 'texture', + }, ]; /** The rows of a clip of a video, set in the Video tab. */ @@ -84,6 +100,10 @@ export function videoTimeOf(bag: PropertyBag): number { return coerceAs(bag[VIDEO_TIME_KEY], 'number', 0); } +export function videoBusOf(bag: PropertyBag): string { + return coerceAs(bag[VIDEO_BUS_KEY], 'asset', ''); +} + export function videoLevelOf(bag: PropertyBag): number { return Math.min(1, Math.max(0, coerceAs(bag[VIDEO_LEVEL_KEY], 'number', 1))); } diff --git a/src/editor/composition/CompositionEditor.ts b/src/editor/composition/CompositionEditor.ts index 0d76d19a..168b6d72 100644 --- a/src/editor/composition/CompositionEditor.ts +++ b/src/editor/composition/CompositionEditor.ts @@ -118,7 +118,7 @@ export class CompositionEditor extends AssetTabEditor { - return { kind, asset, volume: 0, bus: '' }; + return { kind, asset, volume: 0, bus: TO_COMPOSITION_BUS }; } /** Puts the texts where they are wanted, reusing the ones there are and hiding the rest. */ diff --git a/src/editor/debug/documentHooks.ts b/src/editor/debug/documentHooks.ts index c51ccbb2..addab57d 100644 --- a/src/editor/debug/documentHooks.ts +++ b/src/editor/debug/documentHooks.ts @@ -96,11 +96,19 @@ export function documentHooks({ state, editor }: HookContext) { objectScreen: (objectId: string) => editor.compositionEditor.preview.objectScreen(objectId), /** Where the picture draws a sprite right now; null while it is left out, a video without its picture. */ drawn: (objectId: string) => editor.compositionEditor.preview.drawnRect(objectId), - /** Whether its audio items are heard (#418), and the numbers of their playings. */ - sound: () => ({ - playing: editor.compositionEditor.sound.playing, - playings: [...editor.compositionEditor.sound.playings], - }), + /** + * Whether its audio items are heard (#418), the numbers of their playings, how often they were + * started again to meet the picture and how far the sound is ahead of the playhead, in seconds. + */ + sound: () => { + const { sound } = editor.compositionEditor; + return { + playing: sound.playing, + playings: [...sound.playings], + resyncs: sound.resyncs, + ahead: editor.compositionEditor.soundAhead(), + }; + }, }, tile: { current: () => editor.tileEditor.currentAssetId, diff --git a/src/editor/inspector/compositionSection.ts b/src/editor/inspector/compositionSection.ts index 097b0372..ed145f17 100644 --- a/src/editor/inspector/compositionSection.ts +++ b/src/editor/inspector/compositionSection.ts @@ -19,11 +19,14 @@ import { import { fadedItem, heldMove, movedWithClips, resizedItem } from '../../core/model/compositionEdit'; import { findAsset, findAssetOf, type Project } from '../../core/model/Project'; import { FADE_CURVES, findSound, isFadeCurve } from '../../core/model/sound'; +import type { PropertyBag } from '../../core/model/Properties'; +import { FROM_ASSET_LABEL, TO_COMPOSITION_LABEL } from '../../core/properties/buses'; +import { VIDEO_BUS_KEY, VIDEO_CLOCK_KEY, VIDEO_TIME_KEY } from '../../core/properties/videos'; import { DEFAULT_FADE_CURVE } from '../../core/properties/sounds'; import { ITEM_MIN_DB, SOUND_ITEM_MAX_DB } from '../../core/music/song'; import type { PropertyDef } from '../../core/properties/PropertySchema'; -import { Button, formatNumber, type MenuEntry } from '../../ui'; -import { createFieldRow } from './fieldFactory'; +import { Button, formatNumber, Label, type MenuEntry } from '../../ui'; +import { createFieldRow, labelledRow } from './fieldFactory'; import { addNameField, addNote, addTitle, type InspectorHost } from './host'; import { assetLink } from './objectSection'; @@ -40,6 +43,9 @@ const FADE_ROWS = { out: { key: 'composition:fadeOut', curve: 'composition:curveOut', label: 'Fade out' }, } as const; +/** What the Video clock of an item's own sprite says: Free or Driven, the composition sets its time. */ +const DRIVEN_BY_COMPOSITION = 'Driven (by composition)'; + const AUDIO_TITLES: Record = { sound: 'SOUND ITEM', pattern: 'PATTERN ITEM', @@ -62,7 +68,8 @@ const BUS_DEF: PropertyDef = { label: 'Bus', kind: 'asset', assetTypes: ['bus'], - none: 'From asset', + none: FROM_ASSET_LABEL, + toComposition: true, master: true, default: '', }; @@ -188,6 +195,49 @@ export function compositionItemOf( return comp && item ? { comp, item } : undefined; } +/** + * How a composition holds an object of it: `item` for the object an item stands for, whose video the + * composition drives; `content` for one below it. Null outside a composition. + */ +export function compositionHold( + project: Project, + holderId: string, + objectId: string, +): 'item' | 'content' | null { + const comp = findAssetOf(project, holderId, 'composition'); + if (!comp) return null; + return comp.items.some((i) => i.id === objectId) ? 'item' : 'content'; +} + +/** What a composition's sprite reads as: the item's own video driven by the composition. */ +export function heldValues(values: T, hold: ReturnType): T { + return hold === 'item' ? { ...values, [VIDEO_CLOCK_KEY]: 'driven' } : values; +} + +/** + * A row as a composition's sprite shows it: the item's own Video time is the composition's and not + * shown, its Video bus goes into the composition where it picks none. + */ +export function heldDefs( + defs: readonly PropertyDef[], + hold: ReturnType, +): PropertyDef[] { + if (hold === null) return [...defs]; + return defs.flatMap((def): PropertyDef[] => { + if (hold === 'item' && def.key === VIDEO_TIME_KEY) return []; + if (def.key === VIDEO_BUS_KEY && def.kind === 'asset') + return [{ ...def, none: TO_COMPOSITION_LABEL, fromAsset: true }]; + return [def]; + }); +} + +/** In place of the Video clock row of an item's own sprite: the composition sets its time. */ +export function addDrivenClockRow(host: InspectorHost): void { + const { row } = labelledRow('Video clock'); + row.add(new Label(DRIVEN_BY_COMPOSITION), 1); + host.body.add(row); +} + /** * When the object's item lives in the composition, and how far into its video it begins (#417). A * length typed in or an edge moved is the item's own from then on; "Fit to media" hands it back. diff --git a/src/editor/inspector/fieldFactory.ts b/src/editor/inspector/fieldFactory.ts index d1376296..b3abc3c6 100644 --- a/src/editor/inspector/fieldFactory.ts +++ b/src/editor/inspector/fieldFactory.ts @@ -20,7 +20,13 @@ import { soundEntries, } from '../../core/properties/assetRef'; import { NO_FX } from '../../core/properties/assetFx'; -import { MASTER_BUS } from '../../core/properties/buses'; +import { + FROM_ASSET_BUS, + FROM_ASSET_LABEL, + MASTER_BUS, + TO_COMPOSITION_BUS, + TO_COMPOSITION_LABEL, +} from '../../core/properties/buses'; import { keyName } from '../../core/keys'; import { FIRST_TILE_LAYER } from '../../core/model/liveTilemap'; import { type ObjectChoice, objectReference, referencedId } from '../../core/model/objectRefs'; @@ -409,6 +415,8 @@ const REFERENCES: Record = { choices: (project, def) => { const extras = def.kind === 'asset' ? def : undefined; return [ + ...(extras?.fromAsset ? [{ id: FROM_ASSET_BUS, label: FROM_ASSET_LABEL }] : []), + ...(extras?.toComposition ? [{ id: TO_COMPOSITION_BUS, label: TO_COMPOSITION_LABEL }] : []), ...(extras?.master ? [{ id: MASTER_BUS, label: MASTER_LABEL }] : []), ...(extras?.noFx ? [{ id: NO_FX, label: NO_ASSET_LABEL }] : []), ...assetRefChoices(project, extras?.assetTypes).flatMap((asset) => diff --git a/src/editor/inspector/objectSection.ts b/src/editor/inspector/objectSection.ts index ce110107..f74f308b 100644 --- a/src/editor/inspector/objectSection.ts +++ b/src/editor/inspector/objectSection.ts @@ -77,7 +77,16 @@ import { viewOf, } from './objectQueries'; import { componentRow, OFFSET_AXES, offsetKey, POSITION_AXES } from './vectorRows'; -import { addTimingRows, refreshTimingRows, timingSignature } from './compositionSection'; +import { + addDrivenClockRow, + addTimingRows, + compositionHold, + heldDefs, + heldValues, + refreshTimingRows, + timingSignature, +} from './compositionSection'; +import { VIDEO_CLOCK_KEY } from '../../core/properties/videos'; /** * The rows a placed object shows for what it was placed from: a behavior's declared variables but @@ -108,8 +117,11 @@ export function objectSignature( : isLiveInstance(project, holder, object) ? 'instance' : ''; - const values = resolveSceneObjectProps(project, asset, object); - const rows = schemaForObject(project, asset, object).filter((def) => isRowShown(def, values, project)); + const hold = compositionHold(project, holderId, objectId); + const values = heldValues(resolveSceneObjectProps(project, asset, object), hold); + const rows = heldDefs(schemaForObject(project, asset, object), hold).filter((def) => + isRowShown(def, values, project), + ); // A tilemap warns when it is placed more than once, so the number of placements shapes it too. const placed = asset.type === 'tilemap' ? `|placed:${findUsages(project, asset.id).length}` : ''; // A variable turned private leaves the rows, with nothing else about it changed (#257). @@ -260,8 +272,11 @@ export function buildOwnRows(host: InspectorHost, holderId: string, objectId: st const shown = shownObject(project, holderId, objectId); if (!shown) return; const { object, asset } = shown; - const values = resolveSceneObjectProps(project, asset, object); - const own = objectRows(project, asset, object).filter((def) => isRowShown(def, values, project)); + const hold = compositionHold(project, holderId, objectId); + const values = heldValues(resolveSceneObjectProps(project, asset, object), hold); + const own = heldDefs(objectRows(project, asset, object), hold).filter((def) => + isRowShown(def, values, project), + ); host.objectChoices = () => pickableObjects(host.state.project, holderId, objectId); host.objectPicker = objectPicker(host.state, holderId, objectId, host.objectChoices); const behavior = asset.type === 'behavior'; @@ -275,7 +290,9 @@ export function buildOwnRows(host: InspectorHost, holderId: string, objectId: st host, asset.variables.length > 0 ? 'All its variables are private.' : 'The behavior declares no variables.', ); - for (const def of own) propertyRow(host, holderId, objectId, asset, def, values, behavior); + for (const def of own) + if (hold === 'item' && def.key === VIDEO_CLOCK_KEY) addDrivenClockRow(host); + else propertyRow(host, holderId, objectId, asset, def, values, behavior); if (inTab) markObjectRows(host, object, asset, targetOf(project, holderId, objectId)); } diff --git a/src/editor/routing/RoutingEditor.ts b/src/editor/routing/RoutingEditor.ts index eecfcf73..5dc8e25a 100644 --- a/src/editor/routing/RoutingEditor.ts +++ b/src/editor/routing/RoutingEditor.ts @@ -10,7 +10,7 @@ const INFO_WIDTH = 220; /** * The Master routing tab (#351), one per project: a node per sound, instrument, pattern, song, video - * with sound and bus and the master, the links between them where each one goes. Edited as the routing of a song is + * with sound, composition and bus and the master, the links between them where each one goes. Edited as the routing of a song is * (#350); a click on a node shows its asset in the inspector, a double click opens it. */ export class RoutingEditor extends VStack { diff --git a/src/runtime/compositionSound.ts b/src/runtime/compositionSound.ts index 5690148b..dff60659 100644 --- a/src/runtime/compositionSound.ts +++ b/src/runtime/compositionSound.ts @@ -7,6 +7,7 @@ import { compositionTicks, } from '../core/music/compositionAudio'; import { sequenceNeeds } from '../core/music/needs'; +import { secondsOfTicks } from '../core/music/time'; import { LEAD } from '../core/music/Scheduler'; import type { AudioSink } from './systems/Audio'; import { playClip, strikeNote } from './systems/behaviorContext'; @@ -14,16 +15,33 @@ import { playClip, strikeNote } from './systems/behaviorContext'; /** How long the sound of a composition takes to fall silent when it stops. */ const HUSH = 0.02; +/** Seconds the sound may lie off the picture before it is started again where the picture is. */ +export const SLIP = 0.04; + /** * The sound of one composition as it plays (#418): its audio items as playings of the sink, started * from a time of the composition and stopped as a whole. Whoever plays the composition starts it * while its clock runs and stops it while the clock stands, so the two keep together. + * + * The sound runs on the audio clock, the picture on the frames, which count at most `MAX_FRAME` each + * and slower at a time scale below 1. `follow` compares the two every frame and starts the sound + * again where the picture is once they lie more than `SLIP` apart: after a hitch, the sound would + * otherwise stay ahead for good. */ export class CompositionSound { private ids: number[] = []; private started = false; /** What it plays, to tell an edit that changes it from one that does not. */ private heard = ''; + /** The audio clock when it was started, and the second of the composition heard then. */ + private clockAt = 0; + private secondsAt = 0; + /** Seconds of one round where the composition loops, else null. */ + private round: number | null = null; + /** The audio clock at the last `follow`: one that stands holds nothing. */ + private lastClock = 0; + /** How often it was started again to meet the picture (test hook). */ + resyncs = 0; constructor( private readonly audio: AudioSink, @@ -49,6 +67,9 @@ export class CompositionSound { const playings = compositionPlayings(project, comp, routed); this.heard = keyOf(playings); const { length, loop } = compositionSpan(project, comp); + this.clockAt = this.lastClock = this.audio.clock?.() ?? 0; + this.secondsAt = seconds; + this.round = loop ? secondsOfTicks(loop.end - loop.start, COMPOSITION_TEMPO) : null; const sources = { project, audio: this.audio }; for (const { route, sequence, buses } of playings) { const id = this.audio.playSequence( @@ -71,6 +92,41 @@ export class CompositionSound { } } + /** + * Plays the composition as `start` does and, where it plays already, holds it to the picture at + * `seconds`: more than `SLIP` off, it starts again from there. + */ + follow(project: Project, comp: CompositionAsset, seconds: number, routed = true): void { + if (this.started && Math.abs(this.drift(seconds)) > SLIP) { + this.stop(); + this.resyncs++; + } + this.start(project, comp, seconds, routed); + } + + /** + * Seconds the sound is ahead of the picture at `seconds`, behind where negative; 0 with nothing + * heard or an audio clock that stands (a suspended context runs again on its own). + */ + drift(seconds: number): number { + const clock = this.audio.clock?.() ?? 0; + const moved = clock !== this.lastClock; + this.lastClock = clock; + return moved ? (this.aheadOf(seconds) ?? 0) : 0; + } + + /** + * The same without counting as a look at the clock, null with nothing heard (test hook). Round + * and round, both lie in the same round: the nearer way between them counts. + */ + aheadOf(seconds: number): number | null { + const clock = this.audio.clock?.() ?? 0; + if (!this.started || this.ids.length === 0 || clock === 0) return null; + const off = this.secondsAt + (clock - this.clockAt) - seconds; + const round = this.round; + return round && round > 0 ? off - Math.round(off / round) * round : off; + } + /** Whether the composition plays something else now than when it was started. */ changed(project: Project, comp: CompositionAsset, routed = true): boolean { return this.started && keyOf(compositionPlayings(project, comp, routed)) !== this.heard; diff --git a/src/runtime/systems/Audio.ts b/src/runtime/systems/Audio.ts index dbdc4bc4..8a6b6be0 100644 --- a/src/runtime/systems/Audio.ts +++ b/src/runtime/systems/Audio.ts @@ -103,6 +103,11 @@ export interface AudioSink { resetMix(): void; /** Hands new values to the sound FX of a bus of the mix, as `tune` does to a voice. */ tuneMix(bus: string, params: FxParams, time?: number, ease?: EaseShare): void; + /** + * Seconds on the audio clock, 0 before there is one: what a composition holds its sound to its + * picture by. A sink without it keeps no time, and nothing is held to it. + */ + clock?(): number; } /** diff --git a/src/runtime/systems/CompositionSystem.ts b/src/runtime/systems/CompositionSystem.ts index 72332b90..9b4f86c5 100644 --- a/src/runtime/systems/CompositionSystem.ts +++ b/src/runtime/systems/CompositionSystem.ts @@ -14,10 +14,13 @@ import { contentId, placedWithContent } from '../../core/model/instance'; import { findAssetOf } from '../../core/model/Project'; import type { PropertyValue } from '../../core/model/Properties'; import { findVideo, VIDEO_WAIT_MS } from '../../core/model/video'; +import { compositionBusPick } from '../../core/properties/buses'; import { + VIDEO_BUS_KEY, VIDEO_CLOCK_KEY, VIDEO_LEVEL_KEY, VIDEO_TIME_KEY, + videoBusOf, videoDrivenOf, videoLevelOf, } from '../../core/properties/videos'; @@ -142,7 +145,7 @@ export class CompositionSystem implements System { return; } this.show(world, comp, placement, run, frame); - if (runs) run.sound.start(world.project, comp, player.time); + if (runs) run.sound.follow(world.project, comp, player.time); else run.sound.stop(); } @@ -200,6 +203,10 @@ function spawn(world: World, placement: Entity, item: LiveItem): Spawned | null const entities = world.place(placedWithContent(world.project, item.object, id, placement.objectId)); const entity = entities.find((e) => e.objectId === id); if (!entity) return null; + // A sprite that picked no bus is heard on the composition's (#418). + for (const e of entities) + if (e.type === 'sprite' && videoBusOf(e.props) === '') + e.props[VIDEO_BUS_KEY] = compositionBusPick(placement.assetId); // Below the item, a sprite set to be driven by hand keeps its own time. const videos = entities .filter((e) => e.type === 'sprite' && (e === entity || !videoDrivenOf(e.props))) diff --git a/src/viewport/SoundLibrary.ts b/src/viewport/SoundLibrary.ts index 10d421cc..281b46c3 100644 --- a/src/viewport/SoundLibrary.ts +++ b/src/viewport/SoundLibrary.ts @@ -401,8 +401,14 @@ export class SoundLibrary implements AudioSink { private readonly mixVolumes = new Map(); /** The bus of each video file, by the routing of its asset; the master for one not named. */ private videoBuses: ReadonlyMap = new Map(); - /** The sound of every video heard (#402), by its gain: the file it plays, and where it goes. */ - private readonly media = new Map(); + /** + * The sound of every video heard (#402), by its gain: the file it plays, the bus its sprite picked + * (#418) and where it goes. + */ + private readonly media = new Map< + GainNode, + { file: string; routed: boolean; picked?: string; bus: string } + >(); /** The names streamed (#349) and the copies they stream from, by name and by copy. */ private streams: ReadonlyMap = new Map(); /** The cut of each clip, with the copy it is decoded from rather than its whole file (#349). */ @@ -869,17 +875,18 @@ export class SoundLibrary implements AudioSink { return; } this.mix = buses; - if (rerouted) this.rerouteMedia(); + this.rerouteMedia(); if (this.context) await this.applyMix(); } - /** Every video heard along the routing into the bus of its file. */ + /** Every video heard into the bus it goes to now: along the routing of its file, or its pick. */ private rerouteMedia(): void { for (const [gain, heard] of this.media) { - if (!heard.routed) continue; + const bus = this.mediaBus(heard.file, heard.routed, heard.picked); + if (bus === heard.bus) continue; gain.disconnect(); - heard.bus = this.videoBus(heard.file); - gain.connect(this.busInput(heard.bus)); + heard.bus = bus; + gain.connect(this.busInput(bus)); } } @@ -887,6 +894,13 @@ export class SoundLibrary implements AudioSink { return this.videoBuses.get(file) ?? MASTER_MIX; } + /** Where a video is heard: its pick where the mix has that bus – else the master –, else along the routing of its file. */ + private mediaBus(file: string, routed: boolean, picked: string | undefined): string { + if (!routed) return MASTER_MIX; + if (picked === undefined) return this.videoBus(file); + return this.mix.some((bus) => bus.name === picked) ? picked : MASTER_MIX; + } + private async applyMix(): Promise { const names = new Set(this.mix.map((bus) => bus.name)); const waits: Promise[] = []; @@ -1764,19 +1778,20 @@ export class SoundLibrary implements AudioSink { /** * The sound of a video (#402) into the mix, as a voice goes: from now on `element` is heard - * through it and only through it. `routed`, into the bus the routing has for its `file`, else - * straight into the master as a tab's preview (#368). The gain is its volume; hand it to - * `releaseMedia` once the element is let go. Null while a bake has the context. + * through it and only through it. `routed`, into the bus its sprite `picked` (#418) or else the one + * the routing has for its `file`, otherwise straight into the master as a tab's preview (#368). The + * gain is its volume; hand it to `releaseMedia` once the element is let go. Null while a bake has + * the context. */ - mediaInput(element: HTMLMediaElement, file: string, routed: boolean): GainNode | null { + mediaInput(element: HTMLMediaElement, file: string, routed: boolean, picked?: string): GainNode | null { this.audio(); const context = this.live(); if (!context) return null; if (context.state === 'suspended' && !this.held) void context.resume(); const gain = context.createGain(); - const bus = routed ? this.videoBus(file) : MASTER_MIX; + const bus = this.mediaBus(file, routed, picked); context.createMediaElementSource(element).connect(gain).connect(this.busInput(bus)); - this.media.set(gain, { file, routed, bus }); + this.media.set(gain, { file, routed, ...(picked !== undefined && { picked }), bus }); return gain; } diff --git a/src/viewport/VideoLibrary.ts b/src/viewport/VideoLibrary.ts index 1cf610be..b739f68f 100644 --- a/src/viewport/VideoLibrary.ts +++ b/src/viewport/VideoLibrary.ts @@ -384,7 +384,7 @@ export class VideoLibrary { */ private hear(player: Player, routed = true): void { const made = player.gain === null; - player.gain ??= soundLibrary.mediaInput(player.element, player.play.file, routed); + player.gain ??= soundLibrary.mediaInput(player.element, player.play.file, routed, player.play.bus); const level = player.play.volume * (player.drive?.level ?? 1); if (player.gain && made) player.gain.gain.value = level; else if (player.gain && player.level !== level) diff --git a/src/viewport/items.ts b/src/viewport/items.ts index 5b8df838..8eab5c0f 100644 --- a/src/viewport/items.ts +++ b/src/viewport/items.ts @@ -36,7 +36,8 @@ import { type ClipRegion, clipRegions } from '../core/properties/clip'; import { colliderFrame } from '../core/properties/collider'; import { enabledAlongChain } from '../core/properties/enabled'; import { fontOf } from '../core/properties/fonts'; -import { videoDrivenOf } from '../core/properties/videos'; +import { compositionBusPick } from '../core/properties/buses'; +import { VIDEO_BUS_KEY, videoBusOf, videoDrivenOf } from '../core/properties/videos'; import { SORT_WITH_PARENT_KEY } from '../core/properties/PropertySchema'; import { resolveSceneObjectProps } from '../core/properties/resolve'; import { editorTransform } from '../core/properties/transform'; @@ -99,11 +100,13 @@ interface DrawList { * The sprites among `objects` – already laid out, container content and all – whose video is driven: * what a view wears ahead for a composition's coming items. */ -function drivenVideos(project: Project, objects: readonly SceneObject[]): VideoAhead[] { +function drivenVideos(project: Project, objects: readonly SceneObject[], compId = ''): VideoAhead[] { return objects.flatMap((object) => { const asset = findAsset(project, object.assetId); if (asset?.type !== 'sprite') return []; const props = resolveSceneObjectProps(project, asset, object); + // On the bus a run's composition gives its sprites as it spawns them (#418), the same player. + if (compId !== '' && videoBusOf(props) === '') props[VIDEO_BUS_KEY] = compositionBusPick(compId); const texture = resolveTexture(project, props); return texture && isVideoKey(texture.file) && videoDrivenOf(props) ? [{ id: object.id, texture, props }] @@ -130,7 +133,7 @@ function worldVideosAhead(world: World): VideoAhead[] { const objects = compositionAhead(world.project, comp, frame).objects.flatMap((o) => placedWithContent(world.project, o, contentId(placement.objectId, o.id), placement.objectId), ); - return drivenVideos(world.project, objects); + return drivenVideos(world.project, objects, comp.id); }); } diff --git a/tests/core/commands.test.ts b/tests/core/commands.test.ts index ccd82ac4..a4be4db1 100644 --- a/tests/core/commands.test.ts +++ b/tests/core/commands.test.ts @@ -38,6 +38,7 @@ describe('asset commands', () => { videoClock: 'free', videoTime: 0, videoVolume: 1, + videoBus: '', shader: '', }); }); diff --git a/tests/core/composition-audio.test.ts b/tests/core/composition-audio.test.ts index 7fd0db4c..82c3bded 100644 --- a/tests/core/composition-audio.test.ts +++ b/tests/core/composition-audio.test.ts @@ -28,11 +28,12 @@ import { import { carriedClips, itemMedia, mediaMarks } from '../../src/core/model/compositionEdit'; import { findAssetOf } from '../../src/core/model/Project'; import { audioItemBus, compositionPlayings, compositionSpan } from '../../src/core/music/compositionAudio'; -import { MASTER_MIX, mixBusName } from '../../src/core/music/mixer'; +import { MASTER_MIX, mixBusName, mixOf } from '../../src/core/music/mixer'; import { Playbacks, type SequencePlay } from '../../src/core/music/playbacks'; import { LEAD } from '../../src/core/music/Scheduler'; import { newItem } from '../../src/core/music/song'; -import { MASTER_BUS } from '../../src/core/properties/buses'; +import { MASTER_BUS, TO_COMPOSITION_BUS } from '../../src/core/properties/buses'; +import { SLIP } from '../../src/runtime/compositionSound'; import { SILENT_AUDIO } from '../../src/runtime/systems/Audio'; import { addGameSystems } from '../../src/runtime/systems/BehaviorSystem'; import { Input } from '../../src/runtime/systems/Input'; @@ -292,6 +293,14 @@ describe('the sequences of a composition', () => { expect(compositionPlayings(f.project, f.cut(), false).map((p) => p.route)).toEqual([MASTER_MIX]); }); + it('go into the composition’s bus where they pick it, along their asset’s routing where they pick none (#418)', () => { + const f = makeCut(); + f.lay(f.audioItem('a', 'sound', 0, 10), f.audioItem('b', 'pattern', 0, 10, { bus: TO_COMPOSITION_BUS })); + const routes = compositionPlayings(f.project, f.cut()).map((p) => p.route); + expect(routes).toEqual([MASTER_MIX, mixBusName(f.cut().id)]); + expect(mixOf(f.project).find((bus) => bus.name === mixBusName(f.cut().id))?.into).toBe(MASTER_MIX); + }); + it('run as long as the composition, round it where it loops', () => { const f = makeCut(); f.lay(f.audioItem('a', 'sound', 5, 20)); @@ -340,6 +349,9 @@ function recordingAudio() { ...SILENT_AUDIO, started: [] as { id: number; name: string; play: SequencePlay }[], stopped: [] as number[], + /** Its audio clock, moved on by the test: 0 is none, and nothing is held to it. */ + now: 0, + clock: () => sink.now, playSequence: (name: string, play: SequencePlay) => { const id = 100 + sink.started.length; sink.started.push({ id, name, play }); @@ -368,6 +380,14 @@ describe('a composition played in a game', () => { return { audio, world, frame: (times = 1) => Array.from({ length: times }, () => world.tick(0.1)) }; } + /** Frames of 0.1 s, the audio clock moving on by `heard` each. */ + function along(game: ReturnType, times: number, heard = 0.1) { + for (let i = 0; i < times; i++) { + game.audio.now += heard; + game.frame(); + } + } + it('is heard from where its clock stands once it runs, and stops with its end', () => { const f = makeCut(); f.lay(f.audioItem('a', 'sound', 0, 5)); @@ -400,4 +420,36 @@ describe('a composition played in a game', () => { expect(game.audio.started).toHaveLength(2); expect(game.audio.started[1]!.play.from).toBe(Math.round((0.3 + LEAD) * 1000)); }); + + it('starts its sound again where the picture is once a hitch left the two apart', () => { + const f = makeCut(); + f.lay(f.audioItem('a', 'sound', 0, 50)); + const game = run(f); + along(game, 3); + expect(game.audio.started).toHaveLength(1); + // A frame of 0.6 s counts 0.1 for the picture: the sound is 0.5 s ahead. + along(game, 1, 0.6); + expect(game.audio.stopped).toEqual([100]); + expect(game.audio.started).toHaveLength(2); + expect(game.audio.started[1]!.play.from).toBe(Math.round((0.3 + LEAD) * 1000)); + // Then the two keep together, and a few milliseconds apart are no hitch. + along(game, 3); + along(game, 1, 0.1 + SLIP / 2); + expect(game.audio.started).toHaveLength(2); + }); + + it('holds nothing to an audio clock that stands, nor at the turn of a loop', () => { + const f = makeCut(); + f.lay(f.audioItem('a', 'sound', 0, 20)); + f.stack.execute(new SetAssetPropertyCommand(f.cut().id, 'loop', true)); + const game = run(f); + along(game, 2); + along(game, 3, 0); + expect(game.audio.started).toHaveLength(1); + // Round the loop of 2 s three times over, the sound with it. + const loop = run(f); + along(loop, 65); + expect(loop.audio.started).toHaveLength(1); + expect(loop.audio.stopped).toEqual([]); + }); }); diff --git a/tests/core/composition-play.test.ts b/tests/core/composition-play.test.ts index 787821cc..12a88636 100644 --- a/tests/core/composition-play.test.ts +++ b/tests/core/composition-play.test.ts @@ -19,7 +19,14 @@ import { import { contentId, instanceChildId, resolveObjects } from '../../src/core/model/instance'; import { findAsset, findAssetOf, findScene } from '../../src/core/model/Project'; import { createSceneObject } from '../../src/core/model/SceneObject'; -import { VIDEO_CLOCK_KEY, VIDEO_LEVEL_KEY, VIDEO_TIME_KEY } from '../../src/core/properties/videos'; +import { compositionBusPick } from '../../src/core/properties/buses'; +import { parseVideoKey } from '../../src/core/model/video'; +import { + VIDEO_BUS_KEY, + VIDEO_CLOCK_KEY, + VIDEO_LEVEL_KEY, + VIDEO_TIME_KEY, +} from '../../src/core/properties/videos'; import { resolveSceneObjectProps } from '../../src/core/properties/resolve'; import { addGameSystems } from '../../src/runtime/systems/BehaviorSystem'; import type { VideoPictures } from '../../src/runtime/systems/CompositionSystem'; @@ -294,6 +301,45 @@ describe('a composition played in a scene', () => { expect(heard(20)).toBe(0.8); }); + it('hears the video of an item on the composition’s bus where it picks none, ahead too (#418)', () => { + const stage = makeStage(); + director(stage); + const video = new CreateVideoCommand('Intro', FILE, { + width: 320, + height: 180, + duration: 10, + fps: 30, + audio: true, + }); + const film = new CreateAssetCommand('bus', 'Film'); + stage.stack.execute(video); + stage.stack.execute(film); + const screen = new CreateAssetCommand('sprite', 'Screen', { texture: video.asset.id }); + const own = new CreateAssetCommand('sprite', 'Own', { + texture: video.asset.id, + [VIDEO_BUS_KEY]: film.asset.id, + }); + stage.stack.execute(screen); + stage.stack.execute(own); + const item = stage.item(first(stage), screen.asset.id, 0, 20); + const next = stage.item(first(stage), screen.asset.id, 20, 10); + const picked = stage.item(first(stage), own.asset.id, 0, 30); + const run = play(stage); + run.frame(11); + const drawn = (id: string) => + collectWorldItems(run.world).items.find((i) => i.id === contentId(stage.placement, id))?.texture?.file; + const comp = stage.comp().id; + expect(run.at(item)!.props[VIDEO_BUS_KEY]).toBe(compositionBusPick(comp)); + const key = drawn(item); + expect(parseVideoKey(key!)?.bus).toBe(`mix:${comp}`); + // The next one is worn ahead in the player it will be drawn with. + expect(collectWorldItems(run.world).ahead?.[0]?.texture?.file).toBe(key); + expect(run.at(picked)!.props[VIDEO_BUS_KEY]).toBe(film.asset.id); + expect(parseVideoKey(drawn(picked)!)?.bus).toBe(`mix:${film.asset.id}`); + run.frame(10); + expect(drawn(next)).toBe(key); + }); + it('stands still while the view waits for the first picture of an item’s video, 3 s at most', () => { const stage = makeStage(); director(stage); diff --git a/tests/core/master-routing.test.ts b/tests/core/master-routing.test.ts index c4201d30..47719404 100644 --- a/tests/core/master-routing.test.ts +++ b/tests/core/master-routing.test.ts @@ -39,7 +39,10 @@ const QUIET = `${'d4'.repeat(32)}.mp4`; function edited() { const f = makeProject(); const run = (command: Command) => f.stack.execute(new KeepRoutingCommand(command)); - const create = (type: 'bus' | 'instrument' | 'pattern' | 'song' | 'scene', name: string) => { + const create = ( + type: 'bus' | 'instrument' | 'pattern' | 'song' | 'scene' | 'composition', + name: string, + ) => { const command = new CreateAssetCommand(type, name); run(command); return command.asset.id; @@ -390,3 +393,27 @@ describe('the mix of the master routing (#351)', () => { expect(mixOf(f.project).find((bus) => bus.name === mixBusName(kick))?.sends).toEqual([]); }); }); + +describe('a composition in the master routing (#418)', () => { + it('comes linked where the other compositions go, else to the master, and is a bus of the mix', () => { + const f = edited(); + const music = f.create('bus', 'Music'); + const onto = (id: string, bus: string) => { + f.run(new DisconnectCommand(ROUTING_ID, f.project.routing.edges.find((e) => e.from.node === id)!.id)); + f.link(id, bus); + }; + for (const name of ['Theme', 'Battle']) onto(f.create('song', name), music); + const trailer = f.create('composition', 'Trailer'); + expect(f.project.routing.nodes.find((n) => n.id === trailer)?.type).toBe(ROUTED_ASSET); + // A mix of its own: it goes where the songs go only when linked there. + expect(f.linksOf(trailer)).toEqual([MASTER_BUS]); + expect(mixOf(f.project).find((bus) => bus.name === mixBusName(trailer))).toMatchObject({ + volume: 1, + into: MASTER_MIX, + fx: null, + }); + onto(trailer, music); + expect(f.linksOf(f.create('composition', 'Teaser'))).toEqual([music]); + expect(mixOf(f.project).find((bus) => bus.name === mixBusName(trailer))?.into).toBe(mixBusName(music)); + }); +}); diff --git a/tests/core/resolve.test.ts b/tests/core/resolve.test.ts index ff83bd40..fbb7ed78 100644 --- a/tests/core/resolve.test.ts +++ b/tests/core/resolve.test.ts @@ -39,6 +39,7 @@ describe('PropertySchema', () => { videoClock: 'free', videoTime: 0, videoVolume: 1, + videoBus: '', shader: '', }); }); @@ -75,6 +76,7 @@ describe('resolve', () => { videoClock: 'free', videoTime: 0, videoVolume: 1, + videoBus: '', shader: '', }); }); @@ -99,6 +101,7 @@ describe('resolve', () => { videoClock: 'free', videoTime: 0, videoVolume: 1, + videoBus: '', shader: '', }); diff --git a/tests/core/serialize.test.ts b/tests/core/serialize.test.ts index 10fc8046..d15d7f91 100644 --- a/tests/core/serialize.test.ts +++ b/tests/core/serialize.test.ts @@ -186,6 +186,7 @@ describe('serialize', () => { videoClock: 'free', videoTime: 0, videoVolume: 1, + videoBus: '', shader: '', }); }); diff --git a/tests/core/shaders.test.ts b/tests/core/shaders.test.ts index fe9b18a2..2dd261cd 100644 --- a/tests/core/shaders.test.ts +++ b/tests/core/shaders.test.ts @@ -730,6 +730,7 @@ describe('a sprite wearing a shader', () => { 'videoClock', 'videoTime', 'videoVolume', + 'videoBus', 'shader', ]); expect(schemaFor('sprite').find((d) => d.key === SHADER_KEY)).toMatchObject({ @@ -754,6 +755,7 @@ describe('a sprite wearing a shader', () => { 'videoClock', 'videoTime', 'videoVolume', + 'videoBus', 'shader', 'brightness', 'contrast', @@ -762,14 +764,14 @@ describe('a sprite wearing a shader', () => { expect(assetDeviates(f.project, f.sprite())).toBe(true); // A dangling id adds no rows and draws nothing – the picker still shows it. f.sprite().props[SHADER_KEY] = 'gone'; - expect(keys()).toHaveLength(16); + expect(keys()).toHaveLength(17); }); it('gives a placed object the rows of the shader it overrides, filled with the node defaults', () => { const f = wearing(); const id = place(f, f.spriteId, { x: 0, y: 0 }); const object = () => f.object(id); - expect(schemaForObject(f.project, f.sprite(), object())).toHaveLength(16); + expect(schemaForObject(f.project, f.sprite(), object())).toHaveLength(17); f.stack.execute(new SetOverrideCommand(f.sceneId, id, SHADER_KEY, f.s.id)); expect(schemaForObject(f.project, f.sprite(), object()).map((d) => d.key)).toContain('saturation'); expect(resolveSceneObjectProps(f.project, f.sprite(), object())).toMatchObject({ diff --git a/tests/core/texture-commands.test.ts b/tests/core/texture-commands.test.ts index d1fef406..776987df 100644 --- a/tests/core/texture-commands.test.ts +++ b/tests/core/texture-commands.test.ts @@ -77,6 +77,7 @@ describe('CreateAssetCommand with starting props', () => { videoClock: 'free', videoTime: 0, videoVolume: 1, + videoBus: '', shader: '', }); stack.undo(); diff --git a/tests/core/videos.test.ts b/tests/core/videos.test.ts index 2fff571b..a9088717 100644 --- a/tests/core/videos.test.ts +++ b/tests/core/videos.test.ts @@ -3,17 +3,23 @@ * under a key the texture library hands to its player. */ import { describe, expect, it } from 'vitest'; -import { CreateVideoCommand, DuplicateAssetCommand } from '../../src/core/commands/assetCommands'; +import { + CreateAssetCommand, + CreateVideoCommand, + DuplicateAssetCommand, +} from '../../src/core/commands/assetCommands'; import { AddClipCommand, RemoveClipCommand, SetClipPropsCommand } from '../../src/core/commands/clipCommands'; import { CommandError } from '../../src/core/commands/Command'; import type { VideoAsset, VideoClip } from '../../src/core/model/Asset'; import { findAssetOf } from '../../src/core/model/Project'; import { clipsOf } from '../../src/core/model/sound'; -import { resolvePicture } from '../../src/core/model/texture'; +import { resolvePicture, resolveTexture } from '../../src/core/model/texture'; import { + drivenVideoKey, findVideo, isVideoFile, parseVideoKey, + routedVideoKey, posterFileOf, videoKey, videoPlayOf, @@ -21,7 +27,9 @@ import { videoTimeAt, } from '../../src/core/model/video'; import { pictureChoices, textureRefLabel, videoChoices } from '../../src/core/properties/textureRef'; +import { compositionBusPick, FROM_ASSET_BUS, MASTER_BUS } from '../../src/core/properties/buses'; import { + VIDEO_BUS_KEY, VIDEO_END_KEY, VIDEO_LOOP_KEY, VIDEO_START_KEY, @@ -159,6 +167,29 @@ describe('the key a playing video goes by', () => { expect(videoPlayOf({ asset: video(), clip: tail })).toMatchObject({ start: 10, end: Infinity }); }); + it('names the bus a sprite picked, read back, so each bus plays in a player of its own (#418)', () => { + const { f, video } = setup(); + const whole = videoKey(videoPlayOf({ asset: video(), clip: null })); + const film = new CreateAssetCommand('bus', 'Film'); + const trailer = new CreateAssetCommand('composition', 'Trailer'); + f.stack.execute(film); + f.stack.execute(trailer); + const worn = (pick: string) => + resolveTexture(f.project, { texture: video().id, [VIDEO_BUS_KEY]: pick })!.file; + expect(worn('')).toBe(whole); + expect(worn(FROM_ASSET_BUS)).toBe(whole); + expect(worn(MASTER_BUS)).toBe(`${whole}!mix`); + expect(worn(film.asset.id)).toBe(`${whole}!mix:${film.asset.id}`); + expect(worn('gone')).toBe(`${whole}!mix`); + expect(worn(compositionBusPick(trailer.asset.id))).toBe(`${whole}!mix:${trailer.asset.id}`); + expect(parseVideoKey(worn(film.asset.id))).toEqual({ + ...videoPlayOf({ asset: video(), clip: null }), + bus: `mix:${film.asset.id}`, + }); + expect(parseVideoKey(drivenVideoKey(worn(MASTER_BUS), 'game:p~i'))?.bus).toBe('mix'); + expect(routedVideoKey(worn(MASTER_BUS), null)).toBe(whole); + }); + it('is no key for anything else', () => { expect(parseVideoKey(FILE)).toBeNull(); expect(parseVideoKey('gen:abc@1')).toBeNull(); diff --git a/tests/viewport/sound-library.test.ts b/tests/viewport/sound-library.test.ts index 93fa7475..c122c1ce 100644 --- a/tests/viewport/sound-library.test.ts +++ b/tests/viewport/sound-library.test.ts @@ -2595,6 +2595,21 @@ describe('the master and the buses of the project (#325)', () => { expect(library.mediaRoutes()).toEqual([]); }); + it('hears a video on the bus its sprite picked, and on the master once the mix has that bus no more (#418)', async () => { + const library = new SoundLibrary(false); + const video = {} as HTMLMediaElement; + await library.setMix(mix, new Map([['a.mp4', 'mix:b']])); + const picked = library.mediaInput(video, 'a.mp4', true, 'mix:a')!; + const preview = library.mediaInput(video, 'a.mp4', false, 'mix:a')!; + const buses = () => library.mediaRoutes().map((route) => route.bus); + expect(buses()).toEqual(['mix:a', 'mix']); + await library.setMix([mix[0]!, mix[2]!]); + expect(buses()).toEqual(['mix', 'mix']); + await library.setMix(mix); + expect(buses()).toEqual(['mix:a', 'mix']); + for (const gain of [picked, preview]) library.releaseMedia(gain); + }); + it('sends a bus on into another one when the mix says so, and lets one go whose asset is gone', async () => { const library = new SoundLibrary(false); await library.setMix(mix); diff --git a/tools/e2e/68-composition.mjs b/tools/e2e/68-composition.mjs index ee42facd..acfb45c2 100644 --- a/tools/e2e/68-composition.mjs +++ b/tools/e2e/68-composition.mjs @@ -382,7 +382,7 @@ export async function composition(t) { c.items = [ { id: 'cp68-v', track: 'cp68-front', start: 0, length: 120, offset: 0 }, { id: 'cp68-tone', track: 'cp68-back', start: 30, length: 30, offset: 0, - audio: { kind: 'sound', asset: 'cp68-blip', volume: 0, bus: '' } }, + audio: { kind: 'sound', asset: 'cp68-blip', volume: 0, bus: '#composition' } }, ];`); await evaluate(`{ window.cp68Sound = new Promise((done) => { @@ -393,7 +393,15 @@ export async function composition(t) { const time = ${clock}; const tone = __gm.sound.playbacks().find((p) => p.name === 'composition:cp68-cut'); rows.push({ time, clock: __gm.sound.clock(), planned: tone?.clips.length ?? 0, clip: tone?.clips[0] ?? null }); - if (time !== null && time >= 0.6) return done(JSON.stringify(rows)); + if (time !== null && time >= 0.6) { + const buses = __gm.routing.buses(); + rows.at(-1).routes = { + tone: buses.find((b) => b.name === 'play:' + tone?.id)?.into ?? null, + cut: buses.find((b) => b.name === 'mix:cp68-cut')?.into ?? null, + videos: __gm.routing.media().filter((m) => m.routed).map((m) => m.bus), + }; + return done(JSON.stringify(rows)); + } } if (performance.now() > until) return done(JSON.stringify(rows)); requestAnimationFrame(step); @@ -421,6 +429,17 @@ export async function composition(t) { early < 0.05, JSON.stringify({ waited: ran, early, reached, toneClip }), ); + // 68i. The sound item picked the composition's bus, the video item picked none: both go into the + // composition's bus, and that one into the master. + const routes = toneRows.at(-1)?.routes; + check( + 'a sound item and a video item go into the composition’s bus, and that one into the master', + routes?.tone === 'mix:cp68-cut' && + routes.cut === 'mix' && + routes.videos.length > 0 && + routes.videos.every((bus) => bus === 'mix:cp68-cut'), + JSON.stringify(routes), + ); await mutate(` const p = __gm.state.project; diff --git a/tools/e2e/69-composition-tab.mjs b/tools/e2e/69-composition-tab.mjs index f593fda0..edac5494 100644 --- a/tools/e2e/69-composition-tab.mjs +++ b/tools/e2e/69-composition-tab.mjs @@ -9,7 +9,9 @@ * video dragged in wears a sprite the composition keeps to itself. An item dragged in the picture * follows the pointer while the button is down. A band picks items, with Alt clips, and copied clips * are pasted where they fit. A sound dragged in is an audio item, heard while the playhead runs; - * the handle of its fade in drags out a fade its bus rises along. + * the handle of its fade in drags out a fade its bus rises along. A video item's sprite reads as + * driven by the composition and goes into its bus, which has a node in the master routing; after a + * hitch the sound is started again where the playhead is. * * Everything it adds carries a `ct69` id and goes again at the end. */ @@ -533,6 +535,7 @@ export async function compositionTab(t) { soundDrop?.what === 'item' && tone?.audio?.kind === 'sound' && tone.audio.asset === 'ct69-sound' && + tone.audio.bus === '#composition' && tone.start === 10 && tone.length === Math.max(1, Math.round(seconds * 10)) && m.objects.length === 0 && @@ -634,6 +637,70 @@ export async function compositionTab(t) { JSON.stringify({ fadeBox, perFrame, fadeHover, faded, levels }), ); + // 69q. A video dragged in: its sprite's Video clock reads as driven by the composition, its Video + // time – the composition's – is not shown, its Video bus goes into the composition by default. The + // composition has a node of its own in the master routing, linked to the master. + await mutate(` + const c = __gm.state.project.assets.find((a) => a.id === '${CUT}'); + c.objects = []; + c.items = [];`); + await dragAsset(SCREEN, await json(`__gm.composition.laneAt('ct69-t1', 0)`)); + const screenItem = (await model()).items[0]; + // A row is looked for once it is scrolled into view: the inspector is taller than the window. + const videoRows = await json(`({ + driven: __gm.inspectorField('Video clock') !== null && __gm.locate('Driven (by composition)') !== null, + time: __gm.inspectorField('Video time'), + volume: __gm.inspectorField('Video volume') !== null, + bus: __gm.inspectorField('Video bus') !== null && __gm.locate('To composition') !== null, + stored: __gm.state.project.assets.find((a) => a.id === '${CUT}').objects[0]?.overrides ?? null, + })`); + const routing = await json('__gm.routing.graph()'); + const node = routing.nodes.find((n) => n.id === CUT); + const links = routing.edges.filter((e) => e.from.node === CUT).map((e) => e.to.node); + check( + 'a video item reads as driven by the composition, goes into its bus, and the composition is routed', + screenItem !== undefined && + videoRows.driven && + videoRows.time === null && + videoRows.volume && + videoRows.bus && + videoRows.stored?.videoClock === undefined && + node?.params.kind === 'composition' && + JSON.stringify(links) === JSON.stringify(['#master']), + JSON.stringify({ screenItem, videoRows, node, links }), + ); + + // 69r. A hitch of half a second while the tab plays: the playhead counts a tenth of it at most, the + // sound ran on – it is started again where the playhead is, and the two keep together after. + await mutate(` + const c = __gm.state.project.assets.find((a) => a.id === '${CUT}'); + c.objects = []; + c.items = [{ id: 'ct69-long', track: 'ct69-t2', start: 0, length: 100, offset: 0, + audio: { kind: 'sound', asset: 'ct69-sound', volume: 0, bus: '' } }];`); + await evaluate('__gm.composition.scrub(0)'); + const longAt = await json('__gm.composition.playAt()'); + await t.click(longAt.x, longAt.y); + let hitchBefore = null; + for (let i = 0; i < 40 && !(hitchBefore?.playing && hitchBefore.ahead !== null); i++) { + await sleep(50); + hitchBefore = await json('__gm.composition.sound()'); + } + await sleep(300); + hitchBefore = await json('__gm.composition.sound()'); + await evaluate(`{ const until = performance.now() + 500; while (performance.now() < until); null; }`); + await sleep(400); + const hitchAfter = await json('__gm.composition.sound()'); + await t.click(longAt.x, longAt.y); + check( + 'after a hitch the sound of the tab is started again where the playhead is', + hitchBefore?.playing && + Math.abs(hitchBefore.ahead) < 0.06 && + hitchAfter.resyncs > hitchBefore.resyncs && + hitchAfter.playing && + Math.abs(hitchAfter.ahead) < 0.06, + JSON.stringify({ hitchBefore, hitchAfter }), + ); + await showTab('scene'); await mutate(` const p = __gm.state.project;