diff --git a/src/core/commands/assetCommands.ts b/src/core/commands/assetCommands.ts index bf23db23..9c7a2077 100644 --- a/src/core/commands/assetCommands.ts +++ b/src/core/commands/assetCommands.ts @@ -231,6 +231,8 @@ function copyOf(original: Asset, id: string, name: string): Asset { if (copy.type === 'texture' || copy.type === 'pixelart') for (const frame of copy.frames) frame.id = newId(); renewClipIds(copy, new Map()); + // A node names a marker by its id alone (#338), so a copy's markers are told apart. + if (copy.type === 'song') for (const marker of copy.markers ?? []) marker.id = newId(); if (copy.type === 'scene' || copy.type === 'container' || copy.type === 'composition') { const ids = new Map(copy.objects.map((o) => [o.id, newId()])); for (const object of copy.objects) { diff --git a/src/core/graph/NodeContext.ts b/src/core/graph/NodeContext.ts index 08422b5f..01dd0cdd 100644 --- a/src/core/graph/NodeContext.ts +++ b/src/core/graph/NodeContext.ts @@ -3,6 +3,7 @@ import type { CompositionPlayer } from '../model/composition'; import type { Mat } from '../model/matrix'; import type { Position, PropertyBag } from '../model/Properties'; import type { FxParams } from '../soundfx/nodes'; +import type { HeardPosition } from '../music/beats'; import type { RecordingSettings } from '../recording/recording'; import { type ChainTransform, composeAlongChain } from '../model/walk'; import { enabledAlongChain } from '../properties/enabled'; @@ -354,6 +355,11 @@ export interface NodeContext { setBusVolume(bus: string, volume: number, time: number, ease: EaseShare): void; /** The volume a bus is at, `''` the master: what a game set, else that of its asset; 0 for none. */ busVolume(bus: string): number; + /** + * Where a playing song or pattern stands as it is heard (#338): the one numbered `playback`, else + * the one started last of `song` or `pattern`, of every one where both are `''`; null for none. + */ + songPosition(playback: number | null, song: string, pattern: string): HeardPosition | null; /** * Asks for another scene, by asset id. The swap happens once the frame is over – a graph must be * able to finish the flow it is in, and half a frame in the old world and half in the new one is diff --git a/src/core/graph/nodes/data/SongPosition.ts b/src/core/graph/nodes/data/SongPosition.ts new file mode 100644 index 00000000..e8fe93ff --- /dev/null +++ b/src/core/graph/nodes/data/SongPosition.ts @@ -0,0 +1,34 @@ +import type { NodeDef } from '../../NodeRegistry'; +import { asNumber, asString } from '../../values'; +import { patternPort } from '../action/PlayPattern'; +import { songPort } from '../action/PlaySong'; +import { PLAYBACK_INPUT, playbackPort } from '../events/musicEvents'; +import { bool, num } from '../shared'; + +export const SongPosition: NodeDef = { + type: 'SongPosition', + label: 'Song position', + category: 'data', + description: + 'Where a playing song or pattern stands, as it is heard: the output latency of the device taken ' + + 'off, a baked song too, so what is drawn by it keeps to the sound. "bar" and "beat" count from 1 ' + + 'as the ruler does, "fraction" is how far into the beat (0 to below 1), "seconds" how far from ' + + 'the start of the song, back with the loop. Connect "playback" for that playing; else it is the ' + + 'one started last of the Song or the Pattern picked (both picked, either), of every song and ' + + 'pattern where none is. "playing" is off before its first note is heard and after its end; with ' + + 'no such playing everything is 0.', + inputs: [songPort(), patternPort(), playbackPort()], + outputs: [num('bar'), num('beat'), num('fraction'), num('seconds'), bool('playing')], + params: [], + read: (ctx, _params, input) => { + const playback = input(PLAYBACK_INPUT); + const at = ctx.songPosition( + playback === null ? null : asNumber(playback), + asString(input('song')), + asString(input('pattern')), + ); + return at + ? { bar: at.bar, beat: at.beat, fraction: at.fraction, seconds: at.seconds, playing: at.playing } + : { bar: 0, beat: 0, fraction: 0, seconds: 0, playing: false }; + }, +}; diff --git a/src/core/graph/nodes/events/musicEvents.ts b/src/core/graph/nodes/events/musicEvents.ts new file mode 100644 index 00000000..1ff00a71 --- /dev/null +++ b/src/core/graph/nodes/events/musicEvents.ts @@ -0,0 +1,94 @@ +import { textOf } from '../../../model/Properties'; +import type { PropertyDef } from '../../../properties/PropertySchema'; +import type { NodeDef, PortDef } from '../../NodeRegistry'; +import { hiddenOrder, orderParam } from '../../eventOrder'; +import { exec, num, str } from '../shared'; + +/** The input that picks one playing (#338): not typed in, so unconnected it is null and hears every one. */ +export const PLAYBACK_INPUT = 'playback'; +export const playbackPort = (): PortDef => ({ ...num(PLAYBACK_INPUT), inline: false }); + +const songParam: PropertyDef = { + key: 'song', + label: 'Song', + kind: 'asset', + default: '', + assetTypes: ['song'], +}; +const patternParam: PropertyDef = { + key: 'pattern', + label: 'Pattern', + kind: 'asset', + default: '', + assetTypes: ['pattern'], +}; +const markerParam: PropertyDef = { + key: 'marker', + label: 'Marker', + kind: 'asset', + default: '', + assetTypes: ['song'], + markers: true, +}; + +/** A pick left at None is not shown on the node: it hears every one. */ +const hiddenEmpty = + (...keys: string[]) => + (params: Parameters>[0]) => [ + ...keys.filter((key) => textOf(params[key] ?? '') === ''), + ...hiddenOrder(params), + ]; + +const HEARS = + 'It hears every song a Play song plays and every pattern a Play pattern plays, wherever it was ' + + 'played; pick a Song or a Pattern to hear only that one (both picked, either), or connect ' + + '"playback" to hear only that playing. A composition plays none of them. It goes by what is heard, ' + + 'the output latency of the device taken off, a baked song too. A frame comes now and then, so ' + + '"late" says how many seconds ago it was heard; one more than half a beat (at most 0.15 s) ago is ' + + 'not told, and of several passed in one frame only the last.'; + +export const OnBeat: NodeDef = { + type: 'OnBeat', + label: 'On beat', + category: 'event', + description: + 'Runs on every beat of a playing song or pattern, the first beat of a bar too. A beat is a note ' + + 'of the value under the line of the time signature: a quarter in 4/4, an eighth in 6/8. "bar" ' + + 'and "beat" count from 1, as the ruler does, and start over with the loop. ' + + HEARS, + inputs: [playbackPort()], + outputs: [exec(), num('playback'), num('bar'), num('beat'), num('late')], + params: [songParam, patternParam, orderParam], + hiddenParams: hiddenEmpty('song', 'pattern'), +}; + +export const OnBar: NodeDef = { + type: 'OnBar', + label: 'On bar', + category: 'event', + description: + 'Runs at the start of every bar of a playing song or pattern, "bar" counting from 1 as the ruler ' + + 'does; with the loop it starts over at the bar the loop starts in. ' + + HEARS, + inputs: [playbackPort()], + outputs: [exec(), num('playback'), num('bar'), num('late')], + params: [songParam, patternParam, orderParam], + hiddenParams: hiddenEmpty('song', 'pattern'), +}; + +export const OnSongMarker: NodeDef = { + type: 'OnSongMarker', + label: 'On song marker', + category: 'event', + description: + 'Runs when a playing song passes a marker of its ruler, every time round the loop. Pick a Marker ' + + 'to hear only that one, or a Song to hear only its markers; markers on one tick each run it. ' + + '"marker" is the id a marker is known by, "name" what it is called. Connect "playback" to hear ' + + 'only that playing. It goes by what is heard, a baked song too. "late" says how many seconds ago ' + + 'it was heard; one more than half a beat (at most 0.15 s) ago is not told, and of several passed ' + + 'in one frame only the last.', + inputs: [playbackPort()], + outputs: [exec(), num('playback'), str('marker'), str('name'), num('late')], + params: [songParam, markerParam, orderParam], + hiddenParams: hiddenEmpty('song', 'marker'), +}; diff --git a/src/core/graph/nodes/index.ts b/src/core/graph/nodes/index.ts index b2d38400..83fea71f 100644 --- a/src/core/graph/nodes/index.ts +++ b/src/core/graph/nodes/index.ts @@ -125,6 +125,8 @@ import { SubList } from './data/SubList'; import { TextureSize } from './data/TextureSize'; import { OnAnimationFinished } from './events/OnAnimationFinished'; import { OnCompositionFinished } from './events/OnCompositionFinished'; +import { OnBar, OnBeat, OnSongMarker } from './events/musicEvents'; +import { SongPosition } from './data/SongPosition'; import { OnBehavior } from './events/OnBehavior'; import { OnClick } from './events/OnClick'; import { OnMouseWheel } from './events/OnMouseWheel'; @@ -186,6 +188,9 @@ export const BUILT_IN_NODES: readonly NodeDef[] = [ OnMouseWheel, OnAnimationFinished, OnCompositionFinished, + OnBeat, + OnBar, + OnSongMarker, OnBehavior, OnStateEntered, OnStateExited, @@ -323,6 +328,7 @@ export const BUILT_IN_NODES: readonly NodeDef[] = [ StopPattern, PlaySong, StopSong, + SongPosition, SetBusVolume, BusVolume, LoadScene, diff --git a/src/core/music/beats.ts b/src/core/music/beats.ts new file mode 100644 index 00000000..a0dcd130 --- /dev/null +++ b/src/core/music/beats.ts @@ -0,0 +1,218 @@ +/** + * Where a playing of a song or a pattern is heard, and the beats, bars and markers it has passed + * (#338). The audio side keeps each playing's time as an anchor on its clock; a game looks once a + * frame, so whatever it passed since the last look is told late, and too late is not told at all. + */ +import type { SongMarker } from './markers'; +import { playheadAt, type LoopRange } from './sequence'; +import { type Meter, positionOf, secondsOfTicks, ticksOfSeconds, ticksPerBar, ticksPerBeat } from './time'; + +/** The time of one playing: how fast, in what meter, from where, round what, to where. */ +export interface Timeline { + tempo: number; + meter: Meter; + /** Where it started, in ticks. */ + from: number; + /** What it runs round once it gets there; null plays it once to `length`. */ + loop: LoopRange | null; + /** Where it ends when it does not loop, in ticks. */ + length: number; +} + +/** The most a spot passed is told late, and less where half a beat is shorter. */ +export const LATE_MAX = 0.15; + +/** + * How late a spot may be told: half a beat, at most `LATE_MAX`. A frame that slips loses no beat; + * after a long stall the old ones are gone, and no burst of them follows. + */ +export function lateLimit(timeline: Pick): number { + return Math.min(LATE_MAX, secondsOfTicks(ticksPerBeat(timeline.meter), timeline.tempo) / 2); +} + +/** What `Song position` hands on. */ +export interface HeardPosition { + bar: number; + beat: number; + /** How far into the beat, 0 to below 1. */ + fraction: number; + /** Seconds from the start of the song to where it stands; back to the loop's start with it. */ + seconds: number; + tick: number; + /** Heard now: started, and not played to its end. */ + playing: boolean; +} + +/** + * Where a playing stands at `heard` seconds of its playing time, as heard: before it starts (null, or + * less than 0) where it starts, after its end at the end. + */ +export function heardPosition(timeline: Timeline, heard: number | null): HeardPosition { + const { tempo, meter, from, loop, length } = timeline; + let tick = from; + let playing = false; + if (heard !== null && heard >= 0) { + tick = playheadAt(ticksOfSeconds(heard, tempo), from, loop); + playing = !ends(timeline) || tick < length; + if (!playing) tick = length; + } + const beat = ticksPerBeat(meter); + const at = positionOf(tick, meter); + const inBeat = tick - Math.floor(tick / beat) * beat; + return { + bar: at.bar, + beat: at.beat, + fraction: Math.min(Math.max(inBeat / beat, 0), 1 - Number.EPSILON), + seconds: secondsOfTicks(tick, tempo), + tick, + playing, + }; +} + +/** A spot the playhead passed: the tick it is at, and how many seconds ago it was heard. */ +export interface Passed { + tick: number; + late: number; +} + +/** + * The ticks among the playhead's run from `low` to `high` that are spots of a kind. `lowOpen` leaves + * `low` itself out (it was told before), `highOpen` leaves `high` out (the loop's end is its start). + */ +export type SpotsIn = (low: number, high: number, lowOpen: boolean, highOpen: boolean) => number[]; + +/** Every multiple of `step` in the run: the beats or the bars. */ +export function gridSpots(step: number): SpotsIn { + return (low, high, lowOpen, highOpen) => { + const spots: number[] = []; + let n = Math.ceil(low / step); + if (lowOpen && n * step <= low) n++; + for (; n * step < high || (!highOpen && n * step === high); n++) spots.push(n * step); + return spots; + }; +} + +/** Those of `ticks` in the run: the markers. */ +export function tickSpots(ticks: readonly number[]): SpotsIn { + return (low, high, lowOpen, highOpen) => + [...new Set(ticks)].filter((t) => (lowOpen ? t > low : t >= low) && (highOpen ? t < high : t <= high)); +} + +/** Whether it ends rather than going round. */ +function ends(timeline: Timeline): boolean { + return timeline.loop === null; +} + +/** One stretch of playing time the playhead runs straight through: from `start` on, at `head`, to `end`. */ +interface Stretch { + start: number; + head: number; + end: number; +} + +/** The stretch playing time `elapsed` (in ticks) lies in. */ +function stretchAt({ from, loop }: Timeline, elapsed: number): Stretch { + if (!loop || from >= loop.end) return { start: 0, head: from, end: Infinity }; + const toEnd = loop.end - from; + if (elapsed < toEnd) return { start: 0, head: from, end: toEnd }; + const round = loop.end - loop.start; + const start = toEnd + Math.floor((elapsed - toEnd) / round) * round; + return { start, head: loop.start, end: start + round }; +} + +/** The stretch that runs up to where `stretch` begins; null for the first. */ +function stretchBefore({ from, loop }: Timeline, stretch: Stretch): Stretch | null { + if (!loop || stretch.start <= 0) return null; + const toEnd = loop.end - from; + const round = loop.end - loop.start; + const start = stretch.start - round; + return start >= toEnd + ? { start, head: loop.start, end: stretch.start } + : { start: 0, head: from, end: toEnd }; +} + +/** No more stretches than this are looked back through: a loop shorter than a frame is no tune. */ +const MOST_STRETCHES = 16; + +/** + * The spots the playhead passed after `before` up to `now` (seconds of playing time heard, `before` + * null where nothing was heard of it yet), newest first – only those no more than `lateLimit` ago. + * A loop's start is passed each time it comes round; nothing at or after the end of one that ends. + */ +export function passed(timeline: Timeline, before: number | null, now: number, spots: SpotsIn): Passed[] { + const low = Math.max(before ?? -Infinity, now - lateLimit(timeline)); + if (!(now >= 0) || !(now > low)) return []; + const { tempo, length } = timeline; + const elapsedLow = ticksOfSeconds(low, tempo); + const high = ticksOfSeconds(now, tempo); + const found: Passed[] = []; + let stretch: Stretch | null = stretchAt(timeline, high); + let highOpen = false; + for (let n = 0; stretch && n < MOST_STRETCHES; n++) { + // What was heard at `low` was told before; where the run reaches back past a stretch's start, + // that start is among it – the very start of the playing too, before which nothing was heard. + const lowInside = elapsedLow >= stretch.start; + const headLow = stretch.head + ((lowInside ? elapsedLow : stretch.start) - stretch.start); + let headHigh = stretch.head + ((highOpen ? stretch.end : high) - stretch.start); + let open = highOpen; + if (ends(timeline) && headHigh >= length) { + headHigh = length; + open = true; + } + const ticks = spots(headLow, headHigh, lowInside, open).sort((a, b) => b - a); + for (const tick of ticks) { + const at = stretch.start + (tick - stretch.head); + found.push({ tick, late: Math.max(0, now - secondsOfTicks(at, tempo)) }); + } + if (lowInside) break; + stretch = stretchBefore(timeline, stretch); + highOpen = true; + } + return found; +} + +/** A beat passed, as `On beat` tells it. */ +export interface PassedBeat extends Passed { + bar: number; + beat: number; +} + +/** The newest beat passed, if one is still to be told. */ +export function passedBeat(timeline: Timeline, before: number | null, now: number): PassedBeat | null { + const [newest] = passed(timeline, before, now, gridSpots(ticksPerBeat(timeline.meter))); + if (!newest) return null; + const at = positionOf(newest.tick, timeline.meter); + return { ...newest, bar: at.bar, beat: at.beat }; +} + +/** A marker passed, as `On song marker` tells it. */ +export interface PassedMarker extends Passed { + id: string; + name: string; +} + +/** Every marker on a tick passed, newest first; markers sharing a tick in their own order. */ +export function passedMarkers( + timeline: Timeline, + before: number | null, + now: number, + markers: readonly SongMarker[], +): PassedMarker[] { + if (markers.length === 0) return []; + return passed(timeline, before, now, tickSpots(markers.map((m) => m.tick))).flatMap((spot) => + markers.filter((m) => m.tick === spot.tick).map((m) => ({ ...spot, id: m.id, name: m.name })), + ); +} + +/** Of what passed, those passed last: one spot, but every marker on it. */ +export function newestPassed(list: readonly T[]): T[] { + const [first] = list; + return first ? list.filter((p) => p.late === first.late) : []; +} + +/** The newest bar line passed, if one is still to be told. */ +export function passedBar(timeline: Timeline, before: number | null, now: number): PassedBeat | null { + const [newest] = passed(timeline, before, now, gridSpots(ticksPerBar(timeline.meter))); + if (!newest) return null; + return { ...newest, bar: positionOf(newest.tick, timeline.meter).bar, beat: 1 }; +} diff --git a/src/core/music/playbacks.ts b/src/core/music/playbacks.ts index 59df2fb5..21e9997d 100644 --- a/src/core/music/playbacks.ts +++ b/src/core/music/playbacks.ts @@ -131,6 +131,23 @@ export interface Playback { fading: number | null; /** Played from its bake (#340) rather than its notes. */ baked: boolean; + /** How fast, round what and to where it plays, and from where it started (#338). */ + time: PlayingTime; +} + +/** The time of a playing (#338): its tempo, where it started, its loop and its length, in ticks. */ +export interface PlayingTime { + tempo: number; + from: number; + loop: LoopRange | null; + length: number; +} + +/** A playing and how far into its playing time it is heard now, null before it starts (#338). */ +export interface HeardTime extends PlayingTime { + id: number; + name: string; + heard: number | null; } /** Whether a playing goes by `name`: its own, every one of a kind for `kind:`, every one for `''`. */ @@ -185,13 +202,16 @@ export class Playbacks { clips, fading: null, baked: runner instanceof BakedRun, + time: { tempo: play.tempo, from: 0, loop: play.loop, length: play.length }, }; this.list.push(playback); const asked = this.host.now(); void this.host.prepare(play.needs, id).then(() => { if (!this.list.includes(playback) || playback.fading !== null) return; const from = play.from; - runner.start(from === undefined ? 0 : from + ticksOfSeconds(this.host.now() - asked, play.tempo)); + playback.time.from = + from === undefined ? 0 : Math.round(from + ticksOfSeconds(this.host.now() - asked, play.tempo)); + runner.start(playback.time.from); // From the first note heard, which is later where the buses are (#346). const at = runner.startedAt ?? this.host.now() + LEAD; this.host.level(id, play.volume ?? 1, at, Math.max(0, play.fadeIn ?? 0)); @@ -203,6 +223,21 @@ export class Playbacks { return this.current().some((playback) => playback.id === id); } + /** + * Every playing with how far into its playing time it is heard now: the audio clock less the + * output latency, from the moment its first note is heard (#338). With none playing the clock is + * not asked: asking for it makes the audio context, which a page may make only after a gesture. + */ + times(): HeardTime[] { + const list = this.current(); + if (list.length === 0) return []; + const heard = this.host.now() - this.host.latency(); + return list.map(({ id, name, runner, time }) => { + const at = runner.startedAt; + return { id, name, ...time, heard: at === null ? null : heard - at }; + }); + } + /** Those that go by `name` (see `goesBy`); with `fade` they fall silent over that many seconds first. */ stopNamed(name: string, letRing = false, fade = 0): void { this.stopWhere((playback) => goesBy(playback.name, name), letRing, fade); diff --git a/src/core/properties/PropertySchema.ts b/src/core/properties/PropertySchema.ts index eee3c8dc..d8aa3dff 100644 --- a/src/core/properties/PropertySchema.ts +++ b/src/core/properties/PropertySchema.ts @@ -159,6 +159,8 @@ type Extras = K extends 'number' /** Offers `TO_COMPOSITION_BUS`, where empty means the routing of the asset. */ toComposition?: boolean; noFx?: boolean; + /** Offers the markers of the songs, each as "Song / Marker", instead of the songs (#338). */ + markers?: boolean; } : K extends 'texture' ? { uploadedOnly?: boolean; moving?: boolean } diff --git a/src/core/properties/assetRef.ts b/src/core/properties/assetRef.ts index 84e16d47..a8380b90 100644 --- a/src/core/properties/assetRef.ts +++ b/src/core/properties/assetRef.ts @@ -1,7 +1,8 @@ import type { Asset, SoundAsset } from '../model/Asset'; import { type AssetType, isPlaceableType } from '../model/AssetTypes'; import { findAsset, type Project } from '../model/Project'; -import { clipsOf, findSound, soundRefName } from '../model/sound'; +import { clipRefName, clipsOf, findSound, soundRefName } from '../model/sound'; +import { heldSongMarkers } from '../music/markers'; import { listedAssets, markedName } from '../music/embedded'; import { NO_FX } from './assetFx'; import { @@ -29,7 +30,30 @@ export function assetRefLabel(project: Project | null, id: string): string { if (asset) return markedName(asset); // A clip of a sound (#357) reads "Sound / Clip", as a frame does. const clip = project ? findSound(project, id) : null; - return clip ? soundRefName(clip, markedName(clip.asset)) : `${id}?`; + if (clip) return soundRefName(clip, markedName(clip.asset)); + // A marker of a song (#338) reads "Song / Marker", as a clip does. + const marker = project ? findSongMarker(project, id) : null; + return marker ? clipRefName(markedName(marker.song), marker.name) : `${id}?`; +} + +/** The marker of a song an id names (#338), with its song; null where none does. */ +export function findSongMarker(project: Project, id: string): { song: Asset; name: string } | null { + if (id === '') return null; + for (const song of project.assets) { + if (song.type !== 'song') continue; + const marker = heldSongMarkers(song.markers).find((m) => m.id === id); + if (marker) return { song, name: marker.name }; + } + return null; +} + +/** What a marker field offers (#338): the markers of every song, "Theme / Chorus", in the order of their ticks. */ +export function markerChoices(project: Project): { id: string; label: string }[] { + return assetRefChoices(project, ['song']).flatMap((song) => + song.type === 'song' + ? heldSongMarkers(song.markers).map((m) => ({ id: m.id, label: clipRefName(markedName(song), m.name) })) + : [], + ); } /** What a sound field offers (#357): each sound, followed by its clips. */ diff --git a/src/editor/agent/readTools.ts b/src/editor/agent/readTools.ts index 3c5bdda1..70366fab 100644 --- a/src/editor/agent/readTools.ts +++ b/src/editor/agent/readTools.ts @@ -424,11 +424,14 @@ function nodeEntry({ id, def, params }: PaletteEntry): object { }; } -/** A property's row without what only the inspector's drawing needs. */ +/** + * A property's row without what only the inspector's drawing needs. `markers` says the field takes + * the id of a marker of a song (#338), as `get_asset` lists them, not the song's. + */ function propertyRow(def: PropertyDef): object { const { key, label, kind } = def; const row: Record = { key, label, kind, default: def.default }; - for (const extra of ['options', 'min', 'max', 'assetTypes'] as const) { + for (const extra of ['options', 'min', 'max', 'assetTypes', 'markers'] as const) { if (extra in def) row[extra] = (def as Record)[extra]; } return row; diff --git a/src/editor/inspector/fieldFactory.ts b/src/editor/inspector/fieldFactory.ts index 24dc1601..8558fbde 100644 --- a/src/editor/inspector/fieldFactory.ts +++ b/src/editor/inspector/fieldFactory.ts @@ -15,6 +15,7 @@ import { assetRefChoices, assetRefLabel, MASTER_LABEL, + markerChoices, NO_ASSET_LABEL, soundChoices, soundEntries, @@ -415,6 +416,7 @@ const REFERENCES: Record = { label: assetRefLabel, choices: (project, def) => { const extras = def.kind === 'asset' ? def : undefined; + if (extras?.markers) return markerChoices(project); return [ ...(extras?.fromAsset ? [{ id: FROM_ASSET_BUS, label: FROM_ASSET_LABEL }] : []), ...(extras?.toComposition ? [{ id: TO_COMPOSITION_BUS, label: TO_COMPOSITION_LABEL }] : []), diff --git a/src/runtime/audioSwitch.ts b/src/runtime/audioSwitch.ts index 06819aa4..3616312b 100644 --- a/src/runtime/audioSwitch.ts +++ b/src/runtime/audioSwitch.ts @@ -1,6 +1,6 @@ import type { EaseShare } from '../core/model/animation'; import type { MixBus } from '../core/music/mixer'; -import type { SequencePlay } from '../core/music/playbacks'; +import type { HeardTime, SequencePlay } from '../core/music/playbacks'; import type { FxParams, FxPlan, FxVoice } from '../core/soundfx/nodes'; import type { AudioSink, ClipTiming, NoteTiming } from './systems/Audio'; @@ -133,4 +133,9 @@ export class AudioSwitch implements AudioSink { clock(): number { return this.now.clock?.() ?? 0; } + + /** Both sinks' playings: each counts its own on its own clock. */ + playingTimes(): HeardTime[] { + return this.both.flatMap((sink) => sink.playingTimes?.() ?? []); + } } diff --git a/src/runtime/systems/Audio.ts b/src/runtime/systems/Audio.ts index 390589b3..e2784996 100644 --- a/src/runtime/systems/Audio.ts +++ b/src/runtime/systems/Audio.ts @@ -1,5 +1,5 @@ import type { EaseShare } from '../../core/model/animation'; -import type { SequencePlay } from '../../core/music/playbacks'; +import type { HeardTime, SequencePlay } from '../../core/music/playbacks'; import type { MixBus } from '../../core/music/mixer'; import type { FxParams, FxPlan, FxVoice } from '../../core/soundfx/nodes'; @@ -108,6 +108,11 @@ export interface AudioSink { * picture by. A sink without it keeps no time, and nothing is held to it. */ clock?(): number; + /** + * The patterns and songs playing, each with how far into its playing time it is heard now (#338): + * what beats, bars and `Song position` go by. A sink without it plays none to be followed. + */ + playingTimes?(): readonly HeardTime[]; } /** diff --git a/src/runtime/systems/BehaviorSystem.ts b/src/runtime/systems/BehaviorSystem.ts index 2b5a0ad3..e41f79c7 100644 --- a/src/runtime/systems/BehaviorSystem.ts +++ b/src/runtime/systems/BehaviorSystem.ts @@ -24,6 +24,7 @@ import { type Value, } from '../../core/graph'; import { GraphCache } from '../../core/graph/graphIndex'; +import { PLAYBACK_INPUT } from '../../core/graph/nodes/events/musicEvents'; import { PER_CELL_INPUT } from '../../core/graph/nodes/shared'; import { keyName } from '../../core/keys'; import { graphLookup } from '../../core/model/graphLookup'; @@ -53,6 +54,14 @@ import { timerPulses, } from './behaviorPulses'; import { CollisionSystem } from './CollisionSystem'; +import { + barPulses, + beatPulses, + heardPlayings, + markerPulses, + MusicFollower, + type PlaybackOf, +} from './musicPulses'; import type { CollisionSource } from './solids'; import { AnimationSystem } from './AnimationSystem'; import { CompositionSystem, type VideoPictures } from './CompositionSystem'; @@ -223,6 +232,8 @@ export class BehaviorSystem implements System { private readonly breakpoint: Breakpoint | null; /** What was last written to `saves`, so a frame that changed nothing writes nothing. */ private lastSaved = ''; + /** Where each playing of a song or pattern was heard last frame (#338). */ + private readonly music = new MusicFollower(); constructor( private readonly project: Project, @@ -636,6 +647,9 @@ export class BehaviorSystem implements System { const { input, collisions, animations, compositions } = this; const perCell: PerCell = (i, node) => asBoolean(i.interpreter.eventInput(node, PER_CELL_INPUT, contextOf(i))); + // Every frame, paused or not: the music goes on, and what a paused one passed is not told after. + const passed = this.music.follow(heardPlayings(this.project, this.audio)); + const playbackOf: PlaybackOf = (i, node) => i.interpreter.eventInput(node, PLAYBACK_INPUT, contextOf(i)); const kinds: (() => Pulse[])[] = [ () => ticking.flatMap((i) => @@ -650,6 +664,9 @@ export class BehaviorSystem implements System { () => (frozen ? [] : ticking.flatMap((i) => collisionExitPulses(i, collisions, perCell))), () => (frozen ? [] : ticking.flatMap((i) => animationFinishedPulses(i, animations))), () => (frozen ? [] : ticking.flatMap((i) => compositionFinishedPulses(i, compositions))), + () => ticking.flatMap((i) => beatPulses(i, passed, playbackOf)), + () => ticking.flatMap((i) => barPulses(i, passed, playbackOf)), + () => ticking.flatMap((i) => markerPulses(i, passed, playbackOf)), () => pointer.pressed ? pulsesOf(ticking.filter(pointed), 'OnClick', { x: pointer.x, y: pointer.y }) : [], () => diff --git a/src/runtime/systems/behaviorContext.ts b/src/runtime/systems/behaviorContext.ts index 304f4d56..ac7fb892 100644 --- a/src/runtime/systems/behaviorContext.ts +++ b/src/runtime/systems/behaviorContext.ts @@ -42,6 +42,7 @@ import type { World } from '../World'; import type { AudioSink, NoteTiming } from './Audio'; import type { Instance } from './behaviorPulses'; import type { Input } from './Input'; +import { heardPlayings, positionOf } from './musicPulses'; import type { RecordingSink } from '../../core/recording/recording'; import { bodyOf, type CollisionSource, layerHolderOf, NOT_BLOCKED } from './solids'; @@ -235,6 +236,8 @@ export function behaviorContext( const reached = reachedBus(project, bus); return reached ? run.audio.mixVolume(reached) : 0; }, + songPosition: (playback, song, pattern) => + positionOf(heardPlayings(project, run.audio), playback, song, pattern), spawn: (assetId, position, parent) => world.spawn(assetId, position, resolve(parent) ?? null), addJoint: (assetId, owner, connected, at) => { const on = resolve(owner); diff --git a/src/runtime/systems/musicPulses.ts b/src/runtime/systems/musicPulses.ts new file mode 100644 index 00000000..4cfb7fc3 --- /dev/null +++ b/src/runtime/systems/musicPulses.ts @@ -0,0 +1,186 @@ +/** + * What a game hears of its music (#338): the songs and patterns playing, each with its time, and the + * beats, bars and markers each has passed since the frame before – what `On beat`, `On bar`, + * `On song marker` and `Song position` go by. A composition's playings are none of them. + */ +import { asNumber, type GraphNode, type Value } from '../../core/graph'; +import { textOf } from '../../core/model/Properties'; +import { findAssetOf, type Project } from '../../core/model/Project'; +import { + heardPosition, + type HeardPosition, + newestPassed, + passedBar, + type PassedBeat, + type PassedMarker, + passedBeat, + passedMarkers, + type Timeline, +} from '../../core/music/beats'; +import { heldSongMarkers, type SongMarker } from '../../core/music/markers'; +import { patternMeter } from '../../core/music/pattern'; +import { songMeter } from '../../core/music/song'; +import { COMMON_TIME } from '../../core/music/time'; +import type { AudioSink } from './Audio'; +import type { Instance, Pulse } from './behaviorPulses'; + +/** A playing of a song or a pattern, as a game follows it. */ +export interface HeardPlaying { + id: number; + kind: 'song' | 'pattern'; + /** The id of the song or pattern. */ + asset: string; + timeline: Timeline; + /** Seconds into its playing time it is heard now; null before it starts. */ + heard: number | null; + markers: readonly SongMarker[]; +} + +/** The playings of songs and patterns, oldest first. */ +export function heardPlayings(project: Project, audio: AudioSink): HeardPlaying[] { + return (audio.playingTimes?.() ?? []).flatMap((time): HeardPlaying[] => { + const split = time.name.indexOf(':'); + const kind = time.name.slice(0, split); + const id = time.name.slice(split + 1); + if (kind !== 'song' && kind !== 'pattern') return []; + const song = kind === 'song' ? findAssetOf(project, id, 'song') : undefined; + const pattern = kind === 'pattern' ? findAssetOf(project, id, 'pattern') : undefined; + const meter = song ? songMeter(song.props) : pattern ? patternMeter(pattern.props) : COMMON_TIME; + const { tempo, from, loop, length } = time; + return [ + { + id: time.id, + kind, + asset: id, + timeline: { tempo, meter, from, loop, length }, + heard: time.heard, + markers: song ? heldSongMarkers(song.markers) : [], + }, + ]; + }); +} + +/** What a playing passed since the frame before. */ +export interface MusicPassed { + playing: HeardPlaying; + beat: PassedBeat | null; + bar: PassedBeat | null; + markers: PassedMarker[]; +} + +/** + * Follows the playings from frame to frame: each frame what each passed since the last one. One + * first seen is told what it passed no more than the limit ago, its first beat among it. + */ +export class MusicFollower { + private before = new Map(); + + follow(playings: readonly HeardPlaying[]): MusicPassed[] { + const next = new Map(); + const passed = playings.map((playing) => { + const { id, heard, timeline } = playing; + const before = this.before.get(id) ?? null; + if (heard === null) return { playing, beat: null, bar: null, markers: [] }; + next.set(id, heard); + return { + playing, + beat: passedBeat(timeline, before, heard), + bar: passedBar(timeline, before, heard), + markers: passedMarkers(timeline, before, heard, playing.markers), + }; + }); + this.before = next; + return passed; + } +} + +/** The playing a node's "playback" input names; null where it is not connected. */ +export type PlaybackOf = (instance: Instance, node: GraphNode) => Value; + +/** Whether a node hears that playing: by its number, else by the song and the pattern picked. */ +function hears(node: GraphNode, wanted: Value, playing: HeardPlaying, keys: readonly string[]): boolean { + if (wanted !== null) return asNumber(wanted) === playing.id; + const picked = keys.map((key) => ({ key, id: textOf(node.params[key] ?? '') })).filter((p) => p.id !== ''); + return picked.length === 0 || picked.some((p) => p.key === playing.kind && p.id === playing.asset); +} + +export function beatPulses( + instance: Instance, + passed: readonly MusicPassed[], + playbackOf: PlaybackOf, +): Pulse[] { + return instance.interpreter.eventNodes('OnBeat').flatMap((node) => { + const wanted = playbackOf(instance, node); + return passed.flatMap(({ playing, beat }) => + beat && hears(node, wanted, playing, ['song', 'pattern']) + ? [ + { + instance, + node, + outputs: { playback: playing.id, bar: beat.bar, beat: beat.beat, late: beat.late }, + }, + ] + : [], + ); + }); +} + +export function barPulses( + instance: Instance, + passed: readonly MusicPassed[], + playbackOf: PlaybackOf, +): Pulse[] { + return instance.interpreter.eventNodes('OnBar').flatMap((node) => { + const wanted = playbackOf(instance, node); + return passed.flatMap(({ playing, bar }) => + bar && hears(node, wanted, playing, ['song', 'pattern']) + ? [{ instance, node, outputs: { playback: playing.id, bar: bar.bar, late: bar.late } }] + : [], + ); + }); +} + +/** Of the markers a playing passed, the last spot of those the node hears: every marker on it. */ +export function markerPulses( + instance: Instance, + passed: readonly MusicPassed[], + playbackOf: PlaybackOf, +): Pulse[] { + return instance.interpreter.eventNodes('OnSongMarker').flatMap((node) => { + const wanted = playbackOf(instance, node); + const marker = textOf(node.params.marker ?? ''); + return passed.flatMap(({ playing, markers }) => { + if (markers.length === 0 || !hears(node, wanted, playing, ['song'])) return []; + const heard = newestPassed(markers.filter((m) => marker === '' || m.id === marker)); + return heard.map((m) => ({ + instance, + node, + outputs: { playback: playing.id, marker: m.id, name: m.name, late: m.late }, + })); + }); + }); +} + +/** + * Where the playing a `Song position` asks for stands: the one numbered `playback`, else the one + * started last of the song or pattern picked, of every one where none is. + */ +export function positionOf( + playings: readonly HeardPlaying[], + playback: number | null, + song: string, + pattern: string, +): HeardPosition | null { + const picked = [ + ...(song ? [{ kind: 'song', id: song }] : []), + ...(pattern ? [{ kind: 'pattern', id: pattern }] : []), + ]; + const fits = (p: HeardPlaying) => + playback !== null + ? p.id === playback + : picked.length === 0 || picked.some((pick) => pick.kind === p.kind && pick.id === p.asset); + const playing = playings + .filter(fits) + .reduce((last, p) => (!last || p.id > last.id ? p : last), null); + return playing ? heardPosition(playing.timeline, playing.heard) : null; +} diff --git a/src/viewport/SoundLibrary.ts b/src/viewport/SoundLibrary.ts index 3190e84e..6d6128eb 100644 --- a/src/viewport/SoundLibrary.ts +++ b/src/viewport/SoundLibrary.ts @@ -22,6 +22,7 @@ import { BAKE_RATE } from '../core/music/bake'; import { type BakePlay, type BusLevel, + type HeardTime, Playbacks, type SequenceNeeds, type SequencePlay, @@ -1412,6 +1413,11 @@ export class SoundLibrary implements AudioSink { })); } + /** Every pattern and song playing, with how far into it is heard now (#338). */ + playingTimes(): HeardTime[] { + return this.sequences.times(); + } + /** * What goes into the master of a playing, taken off before its level into `to` (#340): what a bake * keeps. Says whether there was such a playing. diff --git a/tests/core/beats.test.ts b/tests/core/beats.test.ts new file mode 100644 index 00000000..a0d984d6 --- /dev/null +++ b/tests/core/beats.test.ts @@ -0,0 +1,135 @@ +/** + * Where a playing is heard, and the beats, bars and markers it passed (#338): a frame looks now and + * then, so what passed since is told late – and too late, or more than the last, not at all. + */ +import { describe, expect, it } from 'vitest'; +import { + heardPosition, + lateLimit, + newestPassed, + passedBar, + passedBeat, + passedMarkers, + type Timeline, +} from '../../src/core/music/beats'; +import { COMMON_TIME } from '../../src/core/music/time'; + +const Q = 480; +const BAR = 4 * Q; +/** 120 BPM in 4/4: a beat is half a second, a bar two. Intro bar 1, then round bar 2. */ +const LOOPED: Timeline = { + tempo: 120, + meter: COMMON_TIME, + from: 0, + loop: { start: BAR, end: 2 * BAR }, + length: 2 * BAR, +}; +const ONCE: Timeline = { ...LOOPED, loop: null }; + +const beatAt = (t: Timeline, before: number | null, now: number) => { + const beat = passedBeat(t, before, now); + return beat && [beat.bar, beat.beat, +beat.late.toFixed(6)]; +}; + +describe('where a playing is heard', () => { + it('counts bar and beat from 1, the fraction into the beat, the seconds into the song', () => { + expect(heardPosition(LOOPED, 0)).toMatchObject({ + bar: 1, + beat: 1, + fraction: 0, + seconds: 0, + playing: true, + }); + expect(heardPosition(LOOPED, 1.25)).toMatchObject({ bar: 1, beat: 3, fraction: 0.5, seconds: 1.25 }); + }); + + it('goes back round the loop with the playhead', () => { + // 5.25 s in: the intro took 2, the loop round 2 more, so a quarter second into the second round. + expect(heardPosition(LOOPED, 5.25)).toMatchObject({ bar: 2, beat: 3, fraction: 0.5, seconds: 3.25 }); + }); + + it('stands where it starts before its first note, at its end after it, playing neither time', () => { + expect(heardPosition(LOOPED, null)).toMatchObject({ bar: 1, beat: 1, playing: false }); + expect(heardPosition(LOOPED, -0.03)).toMatchObject({ bar: 1, beat: 1, playing: false }); + expect(heardPosition(ONCE, 4.5)).toMatchObject({ bar: 3, beat: 1, seconds: 4, playing: false }); + }); + + it('takes the beat from the line of the meter: an eighth in 6/8', () => { + const six: Timeline = { ...ONCE, meter: { beats: 6, unit: 8 } }; + // An eighth is a quarter second at 120 quarters a minute; a bar of six is 1.5 s. + expect(heardPosition(six, 1.625)).toMatchObject({ bar: 2, beat: 1, fraction: 0.5 }); + expect(lateLimit(six)).toBe(0.125); + expect(lateLimit(LOOPED)).toBe(0.15); + }); +}); + +describe('the beats it passed', () => { + it('tells a beat passed since the last frame, with how late', () => { + expect(beatAt(LOOPED, 0.49, 0.51)).toEqual([1, 2, 0.01]); + expect(beatAt(LOOPED, 0.51, 0.53)).toBeNull(); + }); + + it('tells the very first beat to a playing seen for the first time', () => { + expect(beatAt(LOOPED, null, 0.02)).toEqual([1, 1, 0.02]); + expect(beatAt(LOOPED, -0.01, 0.02)).toEqual([1, 1, 0.02]); + }); + + it('still tells a beat a slipped frame passed, but not one more than the limit ago', () => { + expect(beatAt(LOOPED, 0.4, 0.6)).toEqual([1, 2, 0.1]); + expect(beatAt(LOOPED, 0.4, 0.7)).toBeNull(); + }); + + it('tells only the last of several passed in one frame, and nothing after a long stall', () => { + expect(beatAt(LOOPED, 0, 1.6)).toEqual([1, 4, 0.1]); + expect(beatAt(LOOPED, 0, 10.3)).toBeNull(); + }); + + it('tells the loop’s start each time round', () => { + // 4 s in, the intro and one round are over: the head is back at bar 2. + expect(beatAt(LOOPED, 3.99, 4.01)).toEqual([2, 1, 0.01]); + expect(passedBar(LOOPED, 5.99, 6.02)).toMatchObject({ bar: 2, beat: 1 }); + }); + + it('counts beats on the ruler’s grid where the loop starts between them', () => { + const offBeat: Timeline = { ...LOOPED, loop: { start: BAR + Q / 2, end: 2 * BAR } }; + // The second round starts 4 s in at a quarter past bar 2's first beat: its next beat is 0.25 s later. + expect(beatAt(offBeat, 3.99, 4.01)).toBeNull(); + expect(beatAt(offBeat, 4.2, 4.3)).toEqual([2, 2, 0.05]); + }); + + it('tells nothing at the end of one that plays once, nor after it', () => { + expect(beatAt(ONCE, 3.95, 4.05)).toBeNull(); + expect(passedBar(ONCE, 3.95, 4.05)).toBeNull(); + expect(beatAt(ONCE, 3.45, 3.55)).toEqual([2, 4, 0.05]); + }); + + it('tells a bar only on its first beat', () => { + expect(passedBar(LOOPED, 0.49, 0.51)).toBeNull(); + expect(passedBar(LOOPED, 1.99, 2.01)).toMatchObject({ bar: 2, beat: 1 }); + }); +}); + +describe('the markers it passed', () => { + const markers = [ + { id: 'a', name: 'Verse', tick: BAR }, + { id: 'b', name: 'Also verse', tick: BAR }, + { id: 'c', name: 'Turn', tick: 2 * BAR - 10 }, + ]; + + it('tells every marker on the tick passed, each time round the loop', () => { + const passed = passedMarkers(LOOPED, 1.99, 2.01, markers); + expect(passed.map((m) => [m.id, m.name, +m.late.toFixed(6)])).toEqual([ + ['a', 'Verse', 0.01], + ['b', 'Also verse', 0.01], + ]); + expect(passedMarkers(LOOPED, 3.99, 4.01, markers).map((m) => m.id)).toEqual(['a', 'b']); + }); + + it('of markers passed across the loop’s end, keeps the last spot only', () => { + // "Turn" is 10 ticks before the end, heard 3.9896 s in; the loop's start follows at 4 s. + const passed = passedMarkers(LOOPED, 3.98, 4.02, markers); + expect(passed.map((m) => m.id)).toEqual(['a', 'b', 'c']); + expect(newestPassed(passed).map((m) => m.id)).toEqual(['a', 'b']); + expect(newestPassed(passed.filter((m) => m.id === 'c')).map((m) => m.id)).toEqual(['c']); + }); +}); diff --git a/tests/core/event-order.test.ts b/tests/core/event-order.test.ts index d88d3b33..e450828a 100644 --- a/tests/core/event-order.test.ts +++ b/tests/core/event-order.test.ts @@ -104,7 +104,7 @@ describe('the order rule', () => { const events = BUILT_IN_NODES.filter( (def) => def.category === 'event' && def.type !== BEHAVIOR_START && def.type !== ON_BEHAVIOR, ); - expect(events).toHaveLength(19); + expect(events).toHaveLength(22); for (const def of events) { expect( def.params.map((p) => p.key), diff --git a/tests/core/music-events.test.ts b/tests/core/music-events.test.ts new file mode 100644 index 00000000..481e0f1b --- /dev/null +++ b/tests/core/music-events.test.ts @@ -0,0 +1,200 @@ +/** + * A game follows its music (#338): On beat, On bar and On song marker run as the songs and patterns + * it plays pass them, as heard, and Song position says where one stands. + */ +import { describe, expect, it } from 'vitest'; +import { CreateAssetCommand, SetAssetPropertyCommand } from '../../src/core/commands/assetCommands'; +import { SetSongMarkersCommand } from '../../src/core/commands/songCommands'; +import type { HeardTime } from '../../src/core/music/playbacks'; +import { SILENT_AUDIO } from '../../src/runtime/systems/Audio'; +import { heardPlayings, positionOf } from '../../src/runtime/systems/musicPulses'; +import { graphEditor, makeBehavior, makeSpriteScene, place, startWorld } from '../helpers'; + +const Q = 480; +const BAR = 4 * Q; + +/** A song with two markers on bar 2 and one in its middle, a pattern, and a behavior to listen. */ +function listening() { + const f = makeSpriteScene('Box'); + const create = (type: 'song' | 'pattern', name: string) => { + const command = new CreateAssetCommand(type, name); + f.stack.execute(command); + return command.asset.id; + }; + const song = create('song', 'Theme'); + const pattern = create('pattern', 'Riff'); + f.stack.execute(new SetAssetPropertyCommand(pattern, 'sign', '3/4')); + f.stack.execute( + new SetSongMarkersCommand(song, [ + { id: 'verse', name: 'Verse', tick: BAR }, + { id: 'also', name: 'Also', tick: BAR }, + { id: 'half', name: 'Half', tick: BAR / 2 }, + ]), + ); + const behavior = makeBehavior(f, 'Logic'); + place(f, behavior.id, { x: 0, y: 0 }); + const edit = graphEditor(f, behavior); + /** An event that logs `prefix` and two of its outputs, run together. */ + const logs = (type: string, params: Record, prefix: string, a: string, b: string) => { + const event = edit.addNode(type, params); + const join = edit.addNode('Join'); + const first = edit.addNode('Join', { a: prefix }); + edit.connect(event, a, first, 'b'); + edit.connect(first, 'text', join, 'a'); + edit.connect(event, b, join, 'b'); + const log = edit.addNode('Log'); + edit.connect(event, 'exec', log, 'exec'); + edit.connect(join, 'text', log, 'message'); + return event; + }; + return { ...f, song, pattern, edit, logs }; +} + +/** A sink whose playings are heard where a test puts them. */ +function heardAudio(song: string, pattern: string) { + const times: HeardTime[] = [ + { + id: 7, + name: `song:${song}`, + heard: null, + tempo: 120, + from: 0, + loop: { start: BAR, end: 2 * BAR }, + length: 2 * BAR, + }, + { + id: 8, + name: `pattern:${pattern}`, + heard: null, + tempo: 120, + from: 0, + loop: { start: 0, end: 3 * Q }, + length: 3 * Q, + }, + { id: 9, name: 'composition:c', heard: null, tempo: 120, from: 0, loop: null, length: 2 * BAR }, + ]; + return { + audio: { ...SILENT_AUDIO, playingTimes: () => times }, + at(heard: number | null, ...ids: number[]) { + for (const time of times) if (ids.length === 0 || ids.includes(time.id)) time.heard = heard; + }, + }; +} + +describe('On beat and On bar', () => { + it('run on every beat and bar a song plays, as heard, counting from 1', () => { + const f = listening(); + f.logs('OnBeat', { song: f.song }, 'beat ', 'bar', 'beat'); + f.logs('OnBar', { song: f.song }, 'bar ', 'bar', 'late'); + const { audio, at } = heardAudio(f.song, f.pattern); + const { world, frame } = startWorld(f, { behaviors: true, audio }); + frame(); + expect(world.log).toEqual([]); + at(0.01, 7); + frame(); + expect(world.log).toEqual(['beat 11', 'bar 10.01']); + // A frame the audio clock did not move in: nothing new was heard. + frame(); + expect(world.log).toHaveLength(2); + at(0.3, 7); + frame(); + at(0.52, 7); + frame(); + expect(world.log.slice(2)).toEqual(['beat 12']); + // Round the loop: the intro and one round are over 4 s in, the head is back on bar 2. + at(3.98, 7); + frame(); + at(4, 7); + frame(); + expect(world.log.slice(3)).toEqual(['beat 21', 'bar 20']); + }); + + it('tell nothing a stalled frame passed too long ago, and of several only the last', () => { + const f = listening(); + f.logs('OnBeat', {}, 'beat ', 'bar', 'beat'); + const { audio, at } = heardAudio(f.song, f.pattern); + const { world, frame } = startWorld(f, { behaviors: true, audio }); + at(0.6, 7); + frame(); + expect(world.log).toEqual(['beat 12']); + at(1.55, 7); + frame(); + at(3.3, 7); + frame(); + expect(world.log.slice(1)).toEqual(['beat 14']); + }); + + it('hear the song or pattern picked, every one where none is, a composition never', () => { + const f = listening(); + f.logs('OnBeat', {}, 'any ', 'playback', 'beat'); + f.logs('OnBeat', { pattern: f.pattern }, 'pattern ', 'playback', 'beat'); + f.logs('OnBeat', { song: f.song }, 'song ', 'playback', 'beat'); + const { audio, at } = heardAudio(f.song, f.pattern); + const { world, frame } = startWorld(f, { behaviors: true, audio }); + at(0.01); + frame(); + expect(world.log).toEqual(['any 71', 'any 81', 'pattern 81', 'song 71']); + }); + + it('hear only the playing "playback" names where it is connected', () => { + const f = listening(); + const beat = f.logs('OnBeat', {}, 'beat ', 'playback', 'beat'); + const number = f.edit.addNode('Number', { value: 8 }); + f.edit.connect(number, 'value', beat, 'playback'); + const { audio, at } = heardAudio(f.song, f.pattern); + const { world, frame } = startWorld(f, { behaviors: true, audio }); + at(0.01); + frame(); + expect(world.log).toEqual(['beat 81']); + }); +}); + +describe('On song marker', () => { + it('runs for every marker on the tick passed, or for the one picked', () => { + const f = listening(); + f.logs('OnSongMarker', { song: f.song }, 'any ', 'name', 'marker'); + f.logs('OnSongMarker', { marker: 'also' }, 'picked ', 'name', 'playback'); + const { audio, at } = heardAudio(f.song, f.pattern); + const { world, frame } = startWorld(f, { behaviors: true, audio }); + at(0.98, 7); + frame(); + at(1.01, 7); + frame(); + expect(world.log).toEqual(['any Halfhalf']); + at(2.01, 7); + frame(); + expect(world.log.slice(1)).toEqual(['any Verseverse', 'any Alsoalso', 'picked Also7']); + }); +}); + +describe('Song position', () => { + it('answers where the playing picked stands, the one started last where several are', () => { + const f = listening(); + const { audio, at } = heardAudio(f.song, f.pattern); + at(1.25, 7); + at(0.625, 8); + const playings = heardPlayings(f.project, audio); + // The pattern is in 3/4 and runs round its one bar of 1.5 s. + expect(playings.map((p) => [p.kind, p.timeline.meter.beats])).toEqual([ + ['song', 4], + ['pattern', 3], + ]); + expect(positionOf(playings, null, f.song, '')).toMatchObject({ + bar: 1, + beat: 3, + fraction: 0.5, + seconds: 1.25, + playing: true, + }); + // None picked: the pattern, started after the song. + expect(positionOf(playings, null, '', '')).toMatchObject({ beat: 2, fraction: 0.25, seconds: 0.625 }); + expect(positionOf(playings, 7, '', f.pattern)).toMatchObject({ seconds: 1.25 }); + expect(positionOf(playings, 3, '', '')).toBeNull(); + at(1.75, 8); + expect(positionOf(heardPlayings(f.project, audio), null, '', f.pattern)).toMatchObject({ + bar: 1, + beat: 1, + fraction: 0.5, + }); + }); +}); diff --git a/tests/core/node-catalog.test.ts b/tests/core/node-catalog.test.ts index 6d67c00d..40670eee 100644 --- a/tests/core/node-catalog.test.ts +++ b/tests/core/node-catalog.test.ts @@ -2526,6 +2526,28 @@ const CASES: Record = { outputs: { volume: 0.4 }, }, ], + SongPosition: [ + { + name: 'answers where the playing stands, by its number where "playback" is connected (#338)', + context: { + songPosition: (playback: number | null, song: string, pattern: string) => + playback === 4 && song === 's' && pattern === '' + ? { bar: 3, beat: 2, fraction: 0.25, seconds: 5.125, tick: 0, playing: true } + : null, + }, + inputs: { playback: 4, song: 's', pattern: '' }, + outputs: { bar: 3, beat: 2, fraction: 0.25, seconds: 5.125, playing: true }, + }, + { + name: 'asks by the song and pattern alone where "playback" is not connected, all 0 for none', + context: { + songPosition: (playback: number | null) => + playback === null ? null : { bar: 1, beat: 1, fraction: 0, seconds: 0, tick: 0, playing: true }, + }, + inputs: { song: 's' }, + outputs: { bar: 0, beat: 0, fraction: 0, seconds: 0, playing: false }, + }, + ], }; /** Two distinct handles that share a name – `Equals` must still tell them apart. */ @@ -2557,6 +2579,9 @@ const EVENT_NODES: Record = { OnMouseWheel: 'clip-areas.test.ts', OnAnimationFinished: 'animation.test.ts', OnCompositionFinished: 'composition-play.test.ts', + OnBeat: 'music-events.test.ts', + OnBar: 'music-events.test.ts', + OnSongMarker: 'music-events.test.ts', OnStateEntered: 'state-machine.test.ts', OnStateExited: 'state-machine.test.ts', }; diff --git a/tests/core/song-markers.test.ts b/tests/core/song-markers.test.ts index 7f404c09..03aff5a7 100644 --- a/tests/core/song-markers.test.ts +++ b/tests/core/song-markers.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { CreateAssetCommand } from '../../src/core/commands/assetCommands'; +import { CreateAssetCommand, DuplicateAssetCommand } from '../../src/core/commands/assetCommands'; import { SetSongMarkersCommand } from '../../src/core/commands/songCommands'; import type { SongAsset } from '../../src/core/model/Asset'; import { @@ -9,6 +9,7 @@ import { nextMarkerName, withoutMarker, } from '../../src/core/music/markers'; +import { assetRefLabel, findSongMarker, markerChoices } from '../../src/core/properties/assetRef'; import { makeProject, roundTrip } from '../helpers'; function makeSong() { @@ -100,4 +101,32 @@ describe('song markers (#338)', () => { const plain = roundTrip(none.project).assets.find((a) => a.id === none.song().id) as SongAsset; expect('markers' in plain).toBe(false); }); + + it('offers the markers of every song to a marker field, each read "Song / Marker"', () => { + const f = makeSong(); + f.stack.execute( + new SetSongMarkersCommand(f.song().id, [ + { id: 'm2', name: 'Boss', tick: 1920 }, + { id: 'm1', name: 'Intro', tick: 0 }, + ]), + ); + expect(markerChoices(f.project)).toEqual([ + { id: 'm1', label: 'Tune / Intro' }, + { id: 'm2', label: 'Tune / Boss' }, + ]); + expect(assetRefLabel(f.project, 'm2')).toBe('Tune / Boss'); + expect(findSongMarker(f.project, 'm1')?.song.id).toBe(f.song().id); + expect(assetRefLabel(f.project, 'gone')).toBe('gone?'); + }); + + it('gives the markers of a copy of a song ids of their own, so a node names one of them', () => { + const f = makeSong(); + f.stack.execute(new SetSongMarkersCommand(f.song().id, [{ id: 'm1', name: 'Intro', tick: 0 }])); + const duplicate = new DuplicateAssetCommand(f.song().id); + f.stack.execute(duplicate); + const copy = f.project.assets.find((a) => a.id === duplicate.copyId) as SongAsset; + expect(copy.markers?.map((m) => m.name)).toEqual(['Intro']); + expect(copy.markers?.[0]?.id).not.toBe('m1'); + expect(findSongMarker(f.project, 'm1')?.song.id).toBe(f.song().id); + }); }); diff --git a/tests/core/song-play.test.ts b/tests/core/song-play.test.ts index 02e45293..d65d7e02 100644 --- a/tests/core/song-play.test.ts +++ b/tests/core/song-play.test.ts @@ -23,9 +23,10 @@ const Q = 480; const BAR = 4 * Q; /** An audio clock moved by hand, its timers run as it passes them. */ -function fakeHost() { +function fakeHost(baked = false) { const host = { time: 0, + latency: 0, sounding: new Set(), pumps: new Set<() => void>(), loading: [] as (() => void)[], @@ -35,7 +36,7 @@ function fakeHost() { }; const playbacks = new Playbacks({ now: () => host.time, - latency: () => 0, + latency: () => host.latency, playing: (voice) => host.sounding.has(voice), stop: (voices) => { for (const voice of voices) host.sounding.delete(voice); @@ -48,6 +49,7 @@ function fakeHost() { level: (id, gain, at, seconds) => host.levels.push({ id, gain, at, seconds }), after: (at, then) => host.timers.push({ at, then }), closed: (id) => host.closed.push(id), + ...(baked && { playBaked: () => 77 }), }); const run = (seconds: number) => { const end = host.time + seconds; @@ -113,6 +115,63 @@ describe('the songs a game plays', () => { expect(playbacks.current()[0]?.clips[0]).toMatchObject({ sound: 'drum', bus: 'track' }); }); + it('tells how far into it is heard: from its first note, the output latency taken off (#338)', async () => { + const { host, playbacks, loaded } = fakeHost(); + host.latency = 0.02; + playbacks.start(5, 'song:a', song([]), ''); + expect(playbacks.times()).toEqual([ + { + id: 5, + name: 'song:a', + heard: null, + tempo: 120, + from: 0, + loop: { start: BAR, end: 2 * BAR }, + length: 2 * BAR, + }, + ]); + host.time = 1; + await loaded(); + // Its first note is heard LEAD after it starts, and the speakers are 0.02 s behind the clock. + host.time = 3; + expect(playbacks.times()[0]!.heard).toBeCloseTo(3 - 0.02 - (1 + LEAD), 9); + }); + + it('asks no clock while nothing plays: that would make the audio context before a gesture (#338)', () => { + let asked = 0; + const playbacks = new Playbacks({ + now: () => (asked++, 0), + latency: () => 0, + playing: () => false, + stop: () => {}, + wake: () => () => {}, + prepare: () => Promise.resolve(), + level: () => {}, + after: () => {}, + }); + expect(playbacks.times()).toEqual([]); + expect(asked).toBe(0); + }); + + it('tells how far a baked one is heard the same way, from where its bake is heard (#338)', async () => { + const { host, playbacks, loaded } = fakeHost(true); + const bake = { file: 'b.ogg', gain: 1, playback: { end: null, until: 0, round: null } }; + playbacks.start(5, 'song:a', song([], { bake }), ''); + host.time = 1; + await loaded(); + host.time = 2.5; + expect(playbacks.current()[0]?.baked).toBe(true); + expect(playbacks.times()[0]!.heard).toBeCloseTo(2.5 - (1 + LEAD), 9); + }); + + it('starts where it was asked to, as far further in as it waited on its files (#418)', async () => { + const { host, playbacks, loaded } = fakeHost(); + playbacks.start(5, 'composition:c', song([], { from: 960 }), ''); + host.time = 0.5; + await loaded(); + expect(playbacks.times()[0]!.from).toBe(960 + 480); + }); + it('rises from silence to its volume over its fade in, from its first note on', async () => { const { host, playbacks, loaded } = fakeHost(); playbacks.start(5, 'song:a', song([], { volume: 0.5, fadeIn: 3 }), ''); diff --git a/tests/helpers.ts b/tests/helpers.ts index 7e66143d..d4682bbb 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -398,6 +398,7 @@ export function fakeContext(overrides: Partial = {}): FakeContext { busVolume: (bus) => [...ctx.buses].reverse().find((change) => change.bus === bus && change.volume !== undefined)?.volume ?? 1, + songPosition: () => null, loadScene: (sceneId) => ctx.loaded.push(sceneId), spawn: () => null, addJoint: () => null, diff --git a/tools/e2e/65-song.mjs b/tools/e2e/65-song.mjs index 7e3141bf..474492c2 100644 --- a/tools/e2e/65-song.mjs +++ b/tools/e2e/65-song.mjs @@ -936,8 +936,14 @@ export async function song(t) { { ...base, id: 'sg65-i1', track: song.tracks[0].id, kind: 'sound', asset: ${JSON.stringify(SOUND)}, tick: 0, length: ${BAR}, offset: 0 }, { ...base, id: 'sg65-i2', track: song.tracks[1].id, kind: 'pattern', asset: ${JSON.stringify(PATTERN)}, tick: ${BAR}, length: ${BAR}, offset: 0 }, ]; - song.loop = { start: ${BAR}, end: ${2 * BAR} };`); + song.loop = { start: ${BAR}, end: ${2 * BAR} }; + song.markers = [{ id: 'sg65-mark', name: 'Round', tick: ${BAR} }];`); await showTab('scene'); + // 65k2 hears it through these (#338): each beat logs bar, beat, how late it is told and where + // Song position stands then, each marker its name. + const joined = (key, a, b, x) => [ + node(key, 'Join', x, 400, { ...(a !== null && { a }), ...(b !== null && { b }) }), + ]; const rig = await scaffoldBehavior(t, { assetId: 'sg65-logic', name: 'Music 65', @@ -950,10 +956,38 @@ export async function song(t) { node('sg65-play', 'PlaySong', 260, 0, { song: id, volume: 0.8, fadeIn: 1, loop: true }), node('sg65-s', 'OnKeyDown', 0, 200, { key: 's' }), node('sg65-stop', 'StopSong', 260, 200, { song: id, fadeOut: 1 }), + node('sg65-beat', 'OnBeat', 0, 400, { song: id }), + node('sg65-pos', 'SongPosition', 0, 600, { song: id }), + ...joined('sg65-j1', 'beat ', null, 200), + ...joined('sg65-j2', null, '.', 400), + ...joined('sg65-j3', null, null, 600), + ...joined('sg65-j4', null, ' ', 800), + ...joined('sg65-j5', null, null, 1000), + ...joined('sg65-j6', null, ' ', 1200), + ...joined('sg65-j7', null, null, 1400), + node('sg65-log', 'Log', 1600, 400), + node('sg65-marker', 'OnSongMarker', 0, 800, { song: id }), + node('sg65-jm', 'Join', 200, 800, { a: 'marker ' }), + node('sg65-logm', 'Log', 400, 800), ], edges: [ edge('sg65-e3', 'sg65-start', 'exec', 'sg65-play', 'exec'), edge('sg65-e4', 'sg65-s', 'exec', 'sg65-stop', 'exec'), + edge('sg65-b1', 'sg65-beat', 'bar', 'sg65-j1', 'b'), + edge('sg65-b2', 'sg65-j1', 'text', 'sg65-j2', 'a'), + edge('sg65-b3', 'sg65-j2', 'text', 'sg65-j3', 'a'), + edge('sg65-b4', 'sg65-beat', 'beat', 'sg65-j3', 'b'), + edge('sg65-b5', 'sg65-j3', 'text', 'sg65-j4', 'a'), + edge('sg65-b6', 'sg65-j4', 'text', 'sg65-j5', 'a'), + edge('sg65-b7', 'sg65-beat', 'late', 'sg65-j5', 'b'), + edge('sg65-b8', 'sg65-j5', 'text', 'sg65-j6', 'a'), + edge('sg65-b9', 'sg65-j6', 'text', 'sg65-j7', 'a'), + edge('sg65-b10', 'sg65-pos', 'fraction', 'sg65-j7', 'b'), + edge('sg65-b11', 'sg65-beat', 'exec', 'sg65-log', 'exec'), + edge('sg65-b12', 'sg65-j7', 'text', 'sg65-log', 'message'), + edge('sg65-m1', 'sg65-marker', 'name', 'sg65-jm', 'b'), + edge('sg65-m2', 'sg65-marker', 'exec', 'sg65-logm', 'exec'), + edge('sg65-m3', 'sg65-jm', 'text', 'sg65-logm', 'message'), ], }, }); @@ -1004,6 +1038,34 @@ export async function song(t) { resumed === 'running', JSON.stringify({ heldClock, heldLater, resumed }), ); + // 65k2. The game follows the song (#338): On beat runs on every beat as it is heard, 0.5 s apart + // at 120 BPM, bar 2 again after bar 2's last beat as the loop goes round; On song marker + // runs each time round; the pause above held them with the clock. "late" is how long ago + // the beat was heard, and Song position, read then, stands that far into the beat. + const heardLines = async () => + (await json('__gm.console.lines()')) + .map((line) => line.text) + .filter((text) => /(beat|marker) /.test(text)); + const heardSome = await waitFor( + heardLines, + (lines) => lines.filter((l) => l.includes('marker Round')).length >= 2, + 80, + ); + const beats = heardSome.flatMap((text) => { + const m = /beat (\d+)\.(\d+) ([-\d.e]+) ([-\d.e]+)/.exec(text); + return m ? [{ bar: +m[1], beat: +m[2], late: +m[3], fraction: +m[4] }] : []; + }); + const counted = beats.map((b) => `${b.bar}.${b.beat}`); + const expected = ['1.1', '1.2', '1.3', '1.4', '2.1', '2.2', '2.3', '2.4', '2.1', '2.2', '2.3', '2.4']; + const markersHeard = heardSome.filter((text) => text.includes('marker Round')); + check( + 'On beat runs on every beat of the song as heard, round the loop again from bar 2, none lost to the pause; On song marker each round', + counted.length >= 9 && + counted.every((c, i) => c === expected[i]) && + markersHeard.length >= 2 && + beats.every((b) => b.late >= 0 && b.late <= 0.15 && Math.abs(b.fraction * 0.5 - b.late) < 0.02), + JSON.stringify({ beats, markersHeard: markersHeard.length }), + ); await press('s'); const fading = await waitFor(playbacks, (list) => list[0]?.fading !== null); const fadedOut = await waitFor(playbacks, (list) => list.length === 0, 40);