diff --git a/README.md b/README.md index b1594fb2..3be69841 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh The browser suite and the three example games need a machine with a GPU: ```bash -node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2072 checks (GM_STEPS=20 for one step) +node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2073 checks (GM_STEPS=20 for one step) # every tool signs in first – see the Accounts page node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs diff --git a/src/core/model/Asset.ts b/src/core/model/Asset.ts index 0939df1f..b959362a 100644 --- a/src/core/model/Asset.ts +++ b/src/core/model/Asset.ts @@ -8,6 +8,7 @@ import type { AssetType } from './AssetTypes'; import type { PropertyBag, PropertyValue, Vec2 } from './Properties'; import type { Scope } from './Scope'; import type { ObjectGroup, SceneObject } from './SceneObject'; +import type { FadeCurve } from './sound'; export interface AssetBase { id: string; @@ -275,7 +276,9 @@ export interface CompositionTrack { * below 0 the item begins before the video and shows its first picture until then. `fitMedia` says * its length still follows that video, or the default while there is none. Items of a track may * overlap. An item with `audio` spawns no object: it plays a sound, a pattern or a song (#418), and - * `offset` is how far into that it begins. + * `offset` is how far into that it begins. `fadeIn` and `fadeOut` are the frames its sound – what it + * plays, or its video's – rises from silence at its start and falls back at its end, shaped by + * `curveIn` and `curveOut`; absent for none. */ export interface CompositionItem { id: string; @@ -285,6 +288,10 @@ export interface CompositionItem { offset: number; fitMedia?: boolean; audio?: CompositionAudio; + fadeIn?: number; + fadeOut?: number; + curveIn?: FadeCurve; + curveOut?: FadeCurve; } export type CompositionAudioKind = 'sound' | 'pattern' | 'song'; diff --git a/src/core/model/composition.ts b/src/core/model/composition.ts index 3b60091a..c17b06cd 100644 --- a/src/core/model/composition.ts +++ b/src/core/model/composition.ts @@ -14,7 +14,14 @@ import { } from '../properties/compositions'; import { playedSoundById } from '../properties/playedSound'; import { resolveSceneObjectProps } from '../properties/resolve'; -import { VIDEO_CLOCK_KEY, VIDEO_TIME_KEY, videoDrivenOf } from '../properties/videos'; +import { DEFAULT_FADE_CURVE } from '../properties/sounds'; +import { + VIDEO_CLOCK_KEY, + VIDEO_LEVEL_KEY, + VIDEO_TIME_KEY, + videoDrivenOf, + videoLevelOf, +} from '../properties/videos'; import { animationDuration, fpsOf, frameCount, targetNamePath } from './animation'; import { type ReadAt, writeObjectRow } from './animationPreview'; import type { @@ -41,6 +48,7 @@ import { import { findAsset, findAssetOf, type Project } from './Project'; import { booleanOf, type PropertyBag } from './Properties'; import { objectName, type SceneObject } from './SceneObject'; +import { type FadeCurve, fadeGain, isFadeCurve } from './sound'; import { findVideo, videoLengthOf, type VideoRef } from './video'; /** How long an item lasts while it has no video to take its length from. */ @@ -85,6 +93,40 @@ export function setSoundMeasure(measure: (file: string) => number | null): void soundSeconds = measure; } +/** Whether an item has a sound its fades shape: what an audio item plays, or the video its object wears. */ +export function itemSounds(project: Project, comp: CompositionAsset, item: CompositionItem): boolean { + return item.audio !== undefined || itemVideo(project, comp, item) !== null; +} + +/** The fades of an item in whole frames as they work: the fade in no longer than the item, the fade out no longer than the rest. */ +export interface ItemFades { + fadeIn: number; + fadeOut: number; + curveIn: FadeCurve; + curveOut: FadeCurve; +} + +export function itemFades(item: CompositionItem): ItemFades { + const frames = (value: number | undefined, most: number) => + Math.max(0, Math.min(most, Math.round(Number.isFinite(value) ? value! : 0))); + const fadeIn = frames(item.fadeIn, item.length); + return { + fadeIn, + fadeOut: frames(item.fadeOut, item.length - fadeIn), + curveIn: item.curveIn ?? DEFAULT_FADE_CURVE, + curveOut: item.curveOut ?? DEFAULT_FADE_CURVE, + }; +} + +/** How loud an item's sound is `at` frames after its start, its fades considered: 0 to 1. */ +export function fadeLevel(item: CompositionItem, at: number): number { + const { fadeIn, fadeOut, curveIn, curveOut } = itemFades(item); + const rise = fadeIn > 0 && at < fadeIn ? fadeGain(curveIn, at / fadeIn) : 1; + const left = item.length - at; + const fall = fadeOut > 0 && left < fadeOut ? fadeGain(curveOut, left / fadeOut) : 1; + return rise * fall; +} + /** * Seconds the first pass of what an audio item plays lasts: its sound once loaded, its pattern, its * song to the end of its loop – it goes round there, what lies behind is never heard –; 0 while that @@ -234,6 +276,8 @@ export interface LiveItem { time: number; /** Seconds into what its videos play: `time` past the item's offset. */ videoTime: number; + /** How loud its videos are heard, 0 to 1: its fades at this frame. */ + level: number; /** In the order they started; a later one writes over an earlier one. */ clips: ClipAt[]; } @@ -266,7 +310,11 @@ export function compositionAt(project: Project, comp: CompositionAsset, frame: n const at = ((frame - clip.start) / fps) * fpsOf(animation!); playing.push({ clip, animation: animation!, frame: Math.min(at, frameCount(animation!) - 1) }); } - live.push({ item: { item, object, track, time, videoTime: item.offset + time, clips: playing }, order }); + const level = fadeLevel(item, frame - item.start); + live.push({ + item: { item, object, track, time, videoTime: item.offset + time, level, clips: playing }, + order, + }); }); live.sort( (a, b) => b.item.track - a.item.track || a.item.item.start - b.item.item.start || a.order - b.order, @@ -364,7 +412,7 @@ export function heldCompositionTracks(raw: unknown): CompositionTrack[] { /** * Held to what an item may be: the id of an object it spawns and no other item does – an audio item * an id of its own and what it plays –, a track of the composition, a start in whole frames from 0, - * at least a frame long, a finite offset. + * at least a frame long, a finite offset, fades in whole frames above 0 and curves there are. */ export function heldCompositionItems( raw: unknown, @@ -392,6 +440,12 @@ export function heldCompositionItems( }; if (i.fitMedia === true) item.fitMedia = true; if (audio) item.audio = audio; + const fadeIn = Math.round(finite(i.fadeIn)); + const fadeOut = Math.round(finite(i.fadeOut)); + if (fadeIn > 0) item.fadeIn = fadeIn; + if (fadeOut > 0) item.fadeOut = fadeOut; + if (isFadeCurve(i.curveIn)) item.curveIn = i.curveIn; + if (isFadeCurve(i.curveOut)) item.curveOut = i.curveOut; items.push(item); } return items; @@ -449,6 +503,7 @@ function driveVideos(project: Project, holder: CompositionAsset, live: LiveItem) const ref = objectRef(project, holder, object.id); ref?.setOverride(VIDEO_CLOCK_KEY, 'driven'); ref?.setOverride(VIDEO_TIME_KEY, live.videoTime); + if (live.level < 1) ref?.setOverride(VIDEO_LEVEL_KEY, videoLevelOf(props) * live.level); } } diff --git a/src/core/model/compositionEdit.ts b/src/core/model/compositionEdit.ts index 2f00aa84..6cd81f31 100644 --- a/src/core/model/compositionEdit.ts +++ b/src/core/model/compositionEdit.ts @@ -19,6 +19,7 @@ import { DEFAULT_ITEM_SECONDS, fittedLength, itemEnd, + itemFades, itemVideo, } from './composition'; import { findAsset, type Project } from './Project'; @@ -324,6 +325,21 @@ export function trimmedItem( return { ...rest, start, length: item.length - by, offset }; } +/** + * The item with the handle of its fade in or out dragged to `frame`: the fade from its edge to there, + * none where that is at or past the edge, no further than the other fade lets it. A fade of 0 is left + * out of the item. + */ +export function fadedItem(item: CompositionItem, end: 'in' | 'out', frame: number): CompositionItem { + const fades = itemFades(item); + const asked = end === 'in' ? frame - item.start : itemEnd(item) - frame; + const room = item.length - (end === 'in' ? fades.fadeOut : fades.fadeIn); + const frames = Math.max(0, Math.min(room, Math.round(asked))); + const key = end === 'in' ? 'fadeIn' : 'fadeOut'; + const { [key]: _, ...rest } = item; + return frames > 0 ? { ...rest, [key]: frames } : rest; +} + /** * Where an item's video lies in the composition, in frames: where it begins, how long its first pass * is, whether it goes round, and how long every pass after the first is where that differs – a song diff --git a/src/core/music/Scheduler.ts b/src/core/music/Scheduler.ts index 9f9bcdac..84d797f9 100644 --- a/src/core/music/Scheduler.ts +++ b/src/core/music/Scheduler.ts @@ -1,4 +1,5 @@ -import { clipsIn, heldLoop, type LoopRange, notesIn, playheadAt, type Sequence } from './sequence'; +import type { FadeCurve } from '../model/sound'; +import { clipsIn, heldLoop, levelsIn, type LoopRange, notesIn, playheadAt, type Sequence } from './sequence'; import { COMMON_TIME, DEFAULT_TEMPO, @@ -44,6 +45,20 @@ export interface ScheduledClip { tempo: number; } +/** + * One ramp of a bus's level handed to the player (#418): from `at` for `duration` seconds, along the + * part of the ramp from `done` (0 to 1) on – a playhead that lands inside one takes it up there. + */ +export interface ScheduledRamp { + at: number; + duration: number; + bus: string; + gain: number; + rise: boolean; + curve: FadeCurve; + done: number; +} + /** What the scheduler needs of the audio side; a test hands in a clock of its own. */ export interface SchedulerHost { /** Seconds on the audio clock. */ @@ -60,6 +75,8 @@ export interface SchedulerHost { play(note: ScheduledNote): number; /** Starts a clip the same way; a host that plays no clips leaves it out. */ playClip?(clip: ScheduledClip): number; + /** Ramps the level of a bus; a host without levels leaves it out. */ + ramp?(ramp: ScheduledRamp): void; playing(voice: number): boolean; /** Ends these voices; with `letRing` a note falls away in the release of its Envelope. */ stop(voices: readonly number[], letRing?: boolean): void; @@ -216,6 +233,19 @@ export class Scheduler { }); if (voice !== 0) this.voices.push(voice); } + if (this.host.ramp) + for (const due of levelsIn(sequence, this.from, this.loop, this.until, to, this.fresh)) { + const whole = due.length + due.skip; + this.host.ramp({ + at: this.anchor + secondsOfTicks(due.at, this.tempo), + duration: secondsOfTicks(due.length, this.tempo), + bus: due.bus, + gain: due.gain, + rise: due.rise, + curve: due.curve, + done: whole > 0 ? due.skip / whole : 0, + }); + } this.fresh = false; this.until = to; if (!this.loop && playheadAt(to, this.from, null) >= this.end) { diff --git a/src/core/music/compositionAudio.ts b/src/core/music/compositionAudio.ts index 8bd50699..b6c79559 100644 --- a/src/core/music/compositionAudio.ts +++ b/src/core/music/compositionAudio.ts @@ -4,12 +4,12 @@ * lands within a millisecond. A pattern or a song keeps its own tempo inside its item: its notes are * laid out here in the composition's time, each with the tempo a Delay synced to beats counts by. * - * Every item is heard on a bus of its own at its volume, through the sound FX of its pattern or song; - * a song's tracks are buses of that item. One playing goes to one place of the mix, so the items are - * split into one sequence for each place they are sent to. + * Every item is heard on a bus of its own at its volume, its fades ramps of that bus' level, through + * the sound FX of its pattern or song; a song's tracks are buses of that item. One playing goes to + * one place of the mix, so the items are split into one sequence for each place they are sent to. */ import type { CompositionAsset, CompositionItem } from '../model/Asset'; -import { compositionFps, compositionLength, compositionLoops } from '../model/composition'; +import { compositionFps, compositionLength, compositionLoops, itemFades } from '../model/composition'; import { findAssetOf, type Project } from '../model/Project'; import { findSound } from '../model/sound'; import { pickedFx } from '../properties/assetFx'; @@ -23,6 +23,7 @@ import { notesIn, type Sequence, type SequenceClip, + type SequenceLevel, type SequenceNote, } from './sequence'; import { dbToGain, songLoop, songSequence, songTempo } from './song'; @@ -69,15 +70,18 @@ export function compositionPlayings( const route = routed ? playedRoute(project, item.audio.bus, laid.routedAs) : MASTER_MIX; const key = JSON.stringify(route); let playing = byRoute.get(key); - if (!playing) byRoute.set(key, (playing = { route, sequence: { notes: [], clips: [] }, buses: [] })); + if (!playing) + byRoute.set(key, (playing = { route, sequence: { notes: [], clips: [], levels: [] }, buses: [] })); (playing.sequence.notes as SequenceNote[]).push(...laid.notes); (playing.sequence.clips as SequenceClip[]).push(...laid.clips); + (playing.sequence.levels as SequenceLevel[]).push(...laid.levels); playing.buses.push(...laid.buses); } const playings = [...byRoute.values()]; for (const { sequence } of playings) { (sequence.notes as SequenceNote[]).sort((a, b) => a.tick - b.tick || a.note - b.note); (sequence.clips as SequenceClip[]).sort((a, b) => a.tick - b.tick); + (sequence.levels as SequenceLevel[]).sort((a, b) => a.tick - b.tick); } return playings; } @@ -94,6 +98,8 @@ export function compositionSpan( interface Laid { notes: SequenceNote[]; clips: SequenceClip[]; + /** Its fades, on its bus. */ + levels: SequenceLevel[]; buses: BusLevel[]; /** The asset its routing is read from: the sound of a clip, else what it plays. */ routedAs: string; @@ -113,6 +119,7 @@ function laidOut(project: Project, compId: string, item: CompositionItem, fps: n const seconds = secondsOfTicks(end - begin, COMPOSITION_TEMPO); const bus = audioItemBus(compId, item.id); const own: BusLevel = { name: bus, gain: dbToGain(audio.volume), pan: 0 }; + const levels = fadesOf(item, fps, bus, own.gain); if (audio.kind === 'sound') { const ref = findSound(project, audio.asset); if (!ref?.asset.file) return null; @@ -125,7 +132,7 @@ function laidOut(project: Project, compId: string, item: CompositionItem, fps: n volume: 1, bus, }; - return { notes: [], clips: [clip], buses: [own], routedAs: ref.asset.id }; + return { notes: [], clips: [clip], levels, buses: [own], routedAs: ref.asset.id }; } const withFx = (fxId: string, tempo: number): BusLevel => { const fx = fxVoiceOf(project, fxId, tempo); @@ -140,6 +147,7 @@ function laidOut(project: Project, compId: string, item: CompositionItem, fps: n return { notes: heard.notes.map((n) => ({ ...n, tick: begin + n.tick, bus })), clips: [], + levels, buses: [withFx(pickedFx('', pattern.props), tempo)], routedAs: pattern.id, }; @@ -160,11 +168,35 @@ function laidOut(project: Project, compId: string, item: CompositionItem, fps: n return { notes: heard.notes.map((n) => ({ ...n, tick: begin + n.tick, ...(n.bus && { bus: renamed(n.bus) }) })), clips: heard.clips.map((c) => ({ ...c, tick: begin + c.tick, bus: renamed(c.bus) })), + levels, buses: [withFx(pickedFx('', song.props), tempo), ...tracks], routedAs: song.id, }; } +/** + * The fades of an item as ramps of its bus, none without: up from silence at its start – at once + * where it has no fade in, as the round before may have faded it out – and down at its end. + */ +function fadesOf(item: CompositionItem, fps: number, bus: string, gain: number): SequenceLevel[] { + const { fadeIn, fadeOut, curveIn, curveOut } = itemFades(item); + if (fadeIn === 0 && fadeOut === 0) return []; + const tick = (frame: number) => compositionTicks(frame / fps); + const start = tick(item.start); + const end = tick(item.start + item.length); + const rise = { + tick: start, + length: tick(item.start + fadeIn) - start, + bus, + gain, + rise: true, + curve: curveIn, + }; + if (fadeOut === 0) return [rise]; + const fall = tick(item.start + item.length - fadeOut); + return [rise, { tick: fall, length: end - fall, bus, gain, rise: false, curve: curveOut }]; +} + /** * What a pattern or a song plays from `into` seconds on for `seconds`, round its loop, in ticks of * the composition from 0: each note and clip with its own tempo, none ringing past the end. diff --git a/src/core/music/playbacks.ts b/src/core/music/playbacks.ts index 5a38de99..b5209bbf 100644 --- a/src/core/music/playbacks.ts +++ b/src/core/music/playbacks.ts @@ -2,7 +2,14 @@ import type { SendTap } from '../graph/Graph'; import type { FxPlan, FxVoice } from '../soundfx/nodes'; import type { BakePlayback } from './bake'; import type { MixSend } from './mixer'; -import { LEAD, type ScheduledClip, type ScheduledNote, Scheduler, type SchedulerHost } from './Scheduler'; +import { + LEAD, + type ScheduledClip, + type ScheduledNote, + type ScheduledRamp, + Scheduler, + type SchedulerHost, +} from './Scheduler'; import type { LoopRange, Sequence } from './sequence'; import { ticksOfSeconds } from './time'; @@ -55,6 +62,8 @@ export interface SequencePlay { /** Starts one note as the player was asked to, and gives back the number of its voice. */ strike(note: ScheduledNote): number; strikeClip?(clip: ScheduledClip): number; + /** Ramps the level of one of its buses (#418). */ + ramp?(ramp: ScheduledRamp): void; /** Its bake (#340), played in place of its notes and clips where the host can. */ bake?: BakePlay; /** @@ -82,7 +91,7 @@ export interface Runner { stop(letRing?: boolean): void; } -export interface PlaybackHost extends Omit { +export interface PlaybackHost extends Omit { /** Loads what that playing needs and makes its buses ready, their sound FX among them (#346). */ prepare(needs: SequenceNeeds, id: number): Promise; /** How far ahead of its time a note of that playing may be started (see `SchedulerHost.ahead`). */ @@ -149,6 +158,7 @@ export class Playbacks { ahead, play: (note) => (keep(planned, note), play.strike(note)), ...(strikeClip && { playClip: (clip: ScheduledClip) => (keep(clips, clip), strikeClip(clip)) }), + ...(play.ramp && { ramp: play.ramp }), }, () => play.sequence, { tempo: play.tempo, loop: play.loop, end: play.length }, diff --git a/src/core/music/sequence.ts b/src/core/music/sequence.ts index 3354df32..49d9e64d 100644 --- a/src/core/music/sequence.ts +++ b/src/core/music/sequence.ts @@ -4,6 +4,7 @@ * ticks of playing time – `elapsed` since the start, which only ever grows – and of the sequence – * where the playhead stands, which jumps back at the end of a loop. */ +import type { FadeCurve } from '../model/sound'; export interface SequenceNote { /** Where it starts in the sequence, and how long it is held, in ticks. */ @@ -40,9 +41,24 @@ export interface SequenceClip { tempo?: number; } +/** + * A ramp of the level of a bus the sequence plays on: from `tick` for `length` ticks it rises from + * silence to `gain` along `curve`, or falls from there to silence – the fades of a composition's + * item. One 0 ticks long sets the bus to `gain` there. + */ +export interface SequenceLevel { + tick: number; + length: number; + bus: string; + gain: number; + rise: boolean; + curve: FadeCurve; +} + export interface Sequence { notes: readonly SequenceNote[]; clips?: readonly SequenceClip[]; + levels?: readonly SequenceLevel[]; } /** From `start` up to `end`, in ticks of the sequence; `end` is where the playhead jumps back. */ @@ -126,8 +142,30 @@ export function clipsIn( end: number, fresh: boolean, ): DueClip[] { - const clips = sequence.clips ?? []; - const due: DueClip[] = []; + return spansIn(sequence.clips ?? [], from, loop, start, end, fresh); +} + +/** The ramps of the sequence's levels as `clipsIn` finds clips: one the playhead lands inside is taken up there. */ +export function levelsIn( + sequence: Sequence, + from: number, + loop: LoopRange | null, + start: number, + end: number, + fresh: boolean, +): (SequenceLevel & { at: number; skip: number })[] { + return spansIn(sequence.levels ?? [], from, loop, start, end, fresh); +} + +function spansIn( + clips: readonly T[], + from: number, + loop: LoopRange | null, + start: number, + end: number, + fresh: boolean, +): (T & { at: number; skip: number })[] { + const due: (T & { at: number; skip: number })[] = []; if (clips.length === 0) return due; const stretchEnd = loop !== null && from < loop.end ? loop.end : Infinity; let elapsed = start; diff --git a/src/core/properties/videos.ts b/src/core/properties/videos.ts index b145dd8c..468e667d 100644 --- a/src/core/properties/videos.ts +++ b/src/core/properties/videos.ts @@ -21,6 +21,8 @@ export const VIDEO_ROWS: readonly PropertyDef[] = [ */ export const VIDEO_CLOCK_KEY = 'videoClock'; export const VIDEO_TIME_KEY = 'videoTime'; +/** How loud a driven video is heard through this sprite, 0–1, on top of the video's own Volume; a composition's fades set it. */ +export const VIDEO_LEVEL_KEY = 'videoVolume'; export const VIDEO_CLOCK_OPTIONS = [ { value: 'free', label: 'Free' }, @@ -47,6 +49,17 @@ export const SPRITE_VIDEO_ROWS: readonly PropertyDef[] = [ shownFor: { key: VIDEO_CLOCK_KEY, values: ['driven'] }, shownForVideo: 'texture', }, + { + key: VIDEO_LEVEL_KEY, + label: 'Video volume', + kind: 'number', + default: 1, + min: 0, + max: 1, + step: 0.05, + shownFor: { key: VIDEO_CLOCK_KEY, values: ['driven'] }, + shownForVideo: 'texture', + }, ]; /** The rows of a clip of a video, set in the Video tab. */ @@ -71,6 +84,10 @@ export function videoTimeOf(bag: PropertyBag): number { return coerceAs(bag[VIDEO_TIME_KEY], 'number', 0); } +export function videoLevelOf(bag: PropertyBag): number { + return Math.min(1, Math.max(0, coerceAs(bag[VIDEO_LEVEL_KEY], 'number', 1))); +} + /** Start and end of a clip in seconds, `end` 0 for the end of the video. */ export function videoSpanOf(bag: PropertyBag): { start: number; end: number } { const seconds = (key: string) => Math.max(0, coerceAs(bag[key], 'number', 0)); diff --git a/src/editor/composition/CompositionTimeline.ts b/src/editor/composition/CompositionTimeline.ts index 2112e7f4..99df5ea3 100644 --- a/src/editor/composition/CompositionTimeline.ts +++ b/src/editor/composition/CompositionTimeline.ts @@ -28,7 +28,11 @@ import { compositionLength, compositionLoops, fittedLength, + itemEnd, + itemFades, + type ItemFades, itemObject, + itemSounds, itemStands, itemVideo, } from '../../core/model/composition'; @@ -38,6 +42,7 @@ import { clipSpans, droppedLength, droppedSpan, + fadedItem, itemMedia, type ItemMedia, heldMove, @@ -55,7 +60,7 @@ import { placementRejection } from '../../core/model/placement'; import type { Vec2 } from '../../core/model/Properties'; import { findAsset, findAssetOf } from '../../core/model/Project'; import { objectName } from '../../core/model/SceneObject'; -import { findSound } from '../../core/model/sound'; +import { fadeGain, findSound } from '../../core/model/sound'; import { assetRefLabel } from '../../core/properties/assetRef'; import { playedSoundById } from '../../core/properties/playedSound'; import { @@ -93,6 +98,7 @@ import { clipBox, clipsInBand, type CompositionView, + fadeHandles, HEAD_WIDTH, type Hit, hitAt, @@ -123,6 +129,9 @@ const AUDIO_COLORS: Record = { const DEFAULT_VIEW: CompositionView = { panX: 0, pxPerTick: 2, panY: 0, trackHeight: TRACK_HEIGHT }; /** Room at the top of an item for its name; marks of its video keep below it. */ const LABEL_STRIP = 15; +/** The side of a fade handle, and the most line pieces a fade's curve is drawn with. */ +const FADE_HANDLE = 7; +const FADE_STEPS = 32; const CLIP_LABEL = textStyle('small').clone(); CLIP_LABEL.fill = Theme.colors.text; @@ -143,7 +152,9 @@ type Gesture = /** Alt held while dragging items: they go without their clips. */ alone: boolean; } - | { kind: 'resize' | 'trim'; anchor: CompositionItem; item: CompositionItem; media: ItemMedia | null }; + | { kind: 'resize' | 'trim'; anchor: CompositionItem; item: CompositionItem; media: ItemMedia | null } + /** The handle of a fade dragged along the top of its item. */ + | { kind: 'fade'; end: 'in' | 'out'; anchor: CompositionItem; item: CompositionItem }; /** An asset dragged in from the panel: an item or, for an animation, a clip – and whether it fits there. */ interface Drop { @@ -198,8 +209,10 @@ export class CompositionTimeline extends Widget { private assetId = ''; private gesture: Gesture | null = null; private drop: Drop | null = null; - /** The end of an item the pointer would drag; it lights up, and the cursor says so. */ + /** The end or fade handle of an item the pointer would drag; it lights up, and the cursor says so. */ private hover: { id: string; part: Part } | null = null; + /** The item under the pointer: its fade handles show. */ + private pointed: string | null = null; /** Where the pointer was last, in local coordinates: a paste lands there while it is over the lanes. */ private pointer: Vec2 | null = null; /** Sound files whose waveform is on its way. */ @@ -552,7 +565,13 @@ export class CompositionTimeline extends Widget { } private hitAt(asset: CompositionAsset, g: Vec2): Hit | null { - return hitAt(this.view, asset.tracks, this.liveItems(asset), asset.clips, this.lengthOf(asset), g); + const fades = (item: CompositionItem) => this.fadesOf(asset, item); + return hitAt(this.view, asset.tracks, this.liveItems(asset), asset.clips, this.lengthOf(asset), g, fades); + } + + /** The fades of an item that has a sound for them to shape; null for one without. */ + private fadesOf(asset: CompositionAsset, item: CompositionItem): ItemFades | null { + return itemSounds(this.state.project, asset, item) ? itemFades(item) : null; } private onPointerDown(e: FederatedPointerEvent): void { @@ -626,6 +645,11 @@ export class CompositionTimeline extends Widget { return; } if (!picked.includes(item.id) || hit.part !== 'body') this.pickItems([item.id]); + if (hit.part === 'fadeIn' || hit.part === 'fadeOut') { + const end = hit.part === 'fadeIn' ? 'in' : 'out'; + this.gesture = { kind: 'fade', end, anchor: item, item }; + return; + } const media = itemMedia(this.state.project, asset, item); if (hit.part === 'end') this.gesture = { kind: 'resize', anchor: item, item, media }; else if (hit.part === 'start') this.gesture = { kind: 'trim', anchor: item, item, media }; @@ -718,6 +742,9 @@ export class CompositionTimeline extends Widget { gesture.item = resizedItem(gesture.anchor, end - gesture.anchor.start); break; } + case 'fade': + gesture.item = fadedItem(gesture.anchor, gesture.end, frame); + break; case 'trim': gesture.item = trimmedItem( gesture.anchor, @@ -828,11 +855,16 @@ export class CompositionTimeline extends Widget { } case 'resize': case 'trim': + case 'fade': if (JSON.stringify(gesture.item) !== JSON.stringify(gesture.anchor)) this.commit( asset, { items: asset.items.map((i) => (i.id === gesture.anchor.id ? gesture.item : i)) }, - gesture.kind === 'resize' ? 'Resize item' : 'Trim item', + gesture.kind === 'fade' + ? `Set fade ${gesture.end}` + : gesture.kind === 'resize' + ? 'Resize item' + : 'Trim item', ); break; default: @@ -944,15 +976,20 @@ export class CompositionTimeline extends Widget { this.redraw(); } - /** The end of an item under the pointer it can be dragged by: a resize cursor, and the end lights up. */ + /** + * The end or fade handle of an item under the pointer it can be dragged by: a resize cursor, and + * the end or handle lights up. The handles of the item under the pointer show. + */ private followHover(asset: CompositionAsset, local: Vec2, g: Vec2): void { const inside = local.x >= HEAD_WIDTH && local.x <= this.w && local.y >= RULER_HEIGHT && local.y <= this.h; const hit = inside ? this.hitAt(asset, g) : null; const hover = hit?.kind === 'item' && hit.part !== 'body' ? { id: hit.id, part: hit.part } : null; + const pointed = hit?.kind === 'item' ? hit.id : null; const cursor = hover ? 'ew-resize' : 'default'; if (this.cursor !== cursor) this.cursor = cursor; - if (JSON.stringify(hover) === JSON.stringify(this.hover)) return; + if (JSON.stringify(hover) === JSON.stringify(this.hover) && pointed === this.pointed) return; this.hover = hover; + this.pointed = pointed; this.redraw(); } @@ -1139,7 +1176,7 @@ export class CompositionTimeline extends Widget { ); return moved ?? live; } - if (g?.kind === 'resize' || g?.kind === 'trim') + if (g?.kind === 'resize' || g?.kind === 'trim' || g?.kind === 'fade') return live.map((i) => (i.id === g.anchor.id ? g.item : i)); return live; } @@ -1268,6 +1305,8 @@ export class CompositionTimeline extends Widget { }); if (item.audio?.kind === 'sound') this.drawWave(item, box); if (media) this.drawMediaMarks(item, media, box); + const fades = this.fadesOf(asset, item); + if (fades) this.drawFades(item, fades, box); const object = itemObject(asset, item); const placed = object && findAsset(this.state.project, object.assetId); const name = item.audio @@ -1280,8 +1319,71 @@ export class CompositionTimeline extends Widget { labels.push({ text: name, x: Math.max(box.x, HEAD_WIDTH) + 4, y: box.y + 2, w: seen - 8 }); } this.drawOverlaps(shown, rows); + const part = this.hover?.part; const hover = this.hover && boxes.get(this.hover.id); - if (hover && this.hover!.part !== 'body') this.drawGrip(hover, this.hover!.part); + if (hover && (part === 'start' || part === 'end')) this.drawGrip(hover, part); + this.drawFadeHandles(asset, shown, rows, picked); + } + + /** + * The fades of an item as the level its sound rises and falls along, from its bottom edge to its top: + * what lies above is shaded, as the part not heard. + */ + private drawFades(item: CompositionItem, fades: ItemFades, box: Box): void { + const ramp = (from: number, frames: number, rising: boolean, curve: ItemFades['curveIn']) => { + const x0 = HEAD_WIDTH + tickToX(this.view, from); + const x1 = HEAD_WIDTH + tickToX(this.view, from + frames); + const steps = Math.max(1, Math.min(FADE_STEPS, Math.ceil((x1 - x0) / 3))); + const points = Array.from({ length: steps + 1 }, (_, i) => { + const t = i / steps; + const level = fadeGain(curve, rising ? t : 1 - t); + return { x: x0 + (x1 - x0) * t, y: box.y + box.h * (1 - level) }; + }); + const corner = rising ? { x: x0, y: box.y } : { x: x1, y: box.y }; + this.lanes.poly([corner, ...points].flatMap((p) => [p.x, p.y])).fill({ color: 0x000000, alpha: 0.35 }); + this.lanes.moveTo(points[0]!.x, points[0]!.y); + for (const p of points.slice(1)) this.lanes.lineTo(p.x, p.y); + this.lanes.stroke({ width: 1, color: 0xffffff, alpha: 0.7 }); + }; + if (fades.fadeIn > 0) ramp(item.start, fades.fadeIn, true, fades.curveIn); + if (fades.fadeOut > 0) ramp(itemEnd(item) - fades.fadeOut, fades.fadeOut, false, fades.curveOut); + } + + /** + * The handles of the fades on the top edge of an item: of the one under the pointer, the picked + * ones, and every one with a fade; the one the pointer would drag lit. + */ + private drawFadeHandles( + asset: CompositionAsset, + shown: readonly CompositionItem[], + rows: ReadonlyMap, + picked: ReadonlySet, + ): void { + const g = this.gesture; + for (const item of shown) { + const row = rows.get(item.track); + const fades = row !== undefined ? this.fadesOf(asset, item) : null; + const handles = fades && fadeHandles(this.view, row!, item, fades); + if (!handles) continue; + const shows = item.id === this.pointed || picked.has(item.id) || g?.kind === 'fade'; + const y = RULER_HEIGHT + handles.y; + for (const [end, x, frames] of [ + ['in', handles.inX, fades.fadeIn], + ['out', handles.outX, fades.fadeOut], + ] as const) { + if (!shows && frames === 0) continue; + const lit = + (g?.kind === 'fade' && g.anchor.id === item.id && g.end === end) || + (this.hover?.id === item.id && this.hover.part === (end === 'in' ? 'fadeIn' : 'fadeOut')); + const at = HEAD_WIDTH + x; + if (at < HEAD_WIDTH || at > this.w) continue; + const half = FADE_HANDLE / 2; + this.over + .rect(Math.round(at - half), y + 1, FADE_HANDLE, FADE_HANDLE) + .fill(lit ? Theme.colors.accentText : 0xffffff) + .stroke({ width: 1, color: 0x000000, alpha: 0.6 }); + } + } } /** diff --git a/src/editor/composition/compositionView.ts b/src/editor/composition/compositionView.ts index aecaceb3..af6774e8 100644 --- a/src/editor/composition/compositionView.ts +++ b/src/editor/composition/compositionView.ts @@ -43,19 +43,42 @@ export function clipBox(view: CompositionView, row: number, span: { start: numbe }; } -export type Part = 'start' | 'body' | 'end'; +export type Part = 'start' | 'body' | 'end' | 'fadeIn' | 'fadeOut'; /** What lies under a point of the lanes: a clip before an item, as clips are drawn over items. */ export type Hit = { kind: 'clip'; id: string } | { kind: 'item'; id: string; part: Part }; +/** How near a fade handle the pointer takes it, across and down from the top of its item. */ +const FADE_REACH = 6; + +/** + * Where the handles of an item's fades sit, in lane coordinates: on its top edge, at the end of the + * fade in and the start of the fade out. None on an item too narrow to hold both apart from its ends. + */ +export function fadeHandles( + view: CompositionView, + row: number, + item: CompositionItem, + fades: { fadeIn: number; fadeOut: number }, +): { inX: number; outX: number; y: number } | null { + const box = itemBox(view, row, item); + if (box.w < FADE_REACH * 6 || box.h < FADE_REACH * 2) return null; + return { + inX: tickToX(view, item.start + fades.fadeIn), + outX: tickToX(view, item.start + item.length - fades.fadeOut), + y: box.y, + }; +} + /** Items in the order they are drawn: a later start over an earlier one, as in the picture. */ export function stacked(items: readonly T[]): T[] { return [...items].sort((a, b) => a.start - b.start); } /** - * What lies under a point. Of items an edge within reach comes first, the nearest one – also the end - * of one an overlapping item covers, so it can still be dragged –, else the body drawn on top. + * What lies under a point. Of items a fade handle comes first – `fades` says which items have them –, + * then an edge within reach, the nearest one – also the end of one an overlapping item covers, so it + * can still be dragged –, else the body drawn on top. */ export function hitAt( view: CompositionView, @@ -64,6 +87,7 @@ export function hitAt( clips: readonly CompositionClip[], clipLength: (clip: CompositionClip) => number, p: Vec2, + fades: (item: CompositionItem) => { fadeIn: number; fadeOut: number } | null = () => null, ): Hit | null { const rows = new Map(tracks.map((t, i) => [t.id, i])); const inside = (b: Box) => p.x >= b.x && p.x <= b.x + b.w && p.y >= b.y && p.y <= b.y + b.h; @@ -74,6 +98,18 @@ export function hitAt( return { kind: 'clip', id: clip.id }; } const order = stacked(items); + const top = [...order].reverse().find((item) => { + const row = rows.get(item.track); + return row !== undefined && inside(itemBox(view, row, item)); + }); + const faded = top && fades(top); + const handles = faded && fadeHandles(view, rows.get(top.track)!, top, faded); + if (handles && p.y <= handles.y + FADE_REACH * 2) { + const toIn = Math.abs(p.x - handles.inX); + const toOut = Math.abs(p.x - handles.outX); + if (Math.min(toIn, toOut) <= FADE_REACH) + return { kind: 'item', id: top.id, part: toIn <= toOut ? 'fadeIn' : 'fadeOut' }; + } let edge: { hit: Hit; distance: number } | null = null; let body: Hit | null = null; for (let i = order.length - 1; i >= 0; i--) { diff --git a/src/editor/inspector/compositionSection.ts b/src/editor/inspector/compositionSection.ts index 1d2c43bd..097b0372 100644 --- a/src/editor/inspector/compositionSection.ts +++ b/src/editor/inspector/compositionSection.ts @@ -12,11 +12,14 @@ import { compositionFps, fittedLength, isCompositionSprite, + itemEnd, + itemFades, itemVideo, } from '../../core/model/composition'; -import { heldMove, movedWithClips, resizedItem } from '../../core/model/compositionEdit'; +import { fadedItem, heldMove, movedWithClips, resizedItem } from '../../core/model/compositionEdit'; import { findAsset, findAssetOf, type Project } from '../../core/model/Project'; -import { findSound } from '../../core/model/sound'; +import { FADE_CURVES, findSound, isFadeCurve } from '../../core/model/sound'; +import { DEFAULT_FADE_CURVE } from '../../core/properties/sounds'; import { ITEM_MIN_DB, SOUND_ITEM_MAX_DB } from '../../core/music/song'; import type { PropertyDef } from '../../core/properties/PropertySchema'; import { Button, formatNumber, type MenuEntry } from '../../ui'; @@ -32,6 +35,10 @@ const ANIMATION_ROW = 'composition:animation'; const PLAYS_ROW = 'composition:plays'; const VOLUME_ROW = 'composition:volume'; const BUS_ROW = 'composition:bus'; +const FADE_ROWS = { + in: { key: 'composition:fadeIn', curve: 'composition:curveIn', label: 'Fade in' }, + out: { key: 'composition:fadeOut', curve: 'composition:curveOut', label: 'Fade out' }, +} as const; const AUDIO_TITLES: Record = { sound: 'SOUND ITEM', @@ -262,6 +269,57 @@ export function addTimingRows(host: InspectorHost, holderId: string, objectId: s const fit = new Button('Fit to media', { onClick: () => fitToMedia(host, holderId, objectId) }); fit.setEnabled(!(item.fitMedia === true && item.length === fittedLength(host.state.project, comp, item))); host.body.add(fit); + if (video) addFadeRows(host, item, fps, write); +} + +/** + * How long its sound rises from silence at its start and falls back at its end, and along which + * curve; the handles on the item's top edge in the timeline set the same. + */ +function addFadeRows( + host: InspectorHost, + item: CompositionItem, + fps: number, + write: (label: string, edit: (current: CompositionItem) => CompositionItem) => void, +): void { + const fades = itemFades(item); + for (const end of ['in', 'out'] as const) { + const { key, curve, label } = FADE_ROWS[end]; + const frames = end === 'in' ? fades.fadeIn : fades.fadeOut; + const length = createFieldRow(host, seconds(key, label), frames / fps, { + onInput: () => undefined, + onChange: (value) => + write(`Set fade ${end}`, (i) => { + const by = Math.round(Number(value) * fps) || 0; + return fadedItem(i, end, end === 'in' ? i.start + by : itemEnd(i) - by); + }), + }); + const shape = createFieldRow( + host, + { + key: curve, + label: `${label} curve`, + kind: 'enum', + default: DEFAULT_FADE_CURVE, + options: FADE_CURVES, + }, + end === 'in' ? fades.curveIn : fades.curveOut, + { + onInput: () => undefined, + onChange: (value) => { + if (!isFadeCurve(value)) return; + write(`Set fade ${end} curve`, (i) => ({ ...i, [end === 'in' ? 'curveIn' : 'curveOut']: value })); + }, + }, + ); + for (const [rowKey, row] of [ + [key, length], + [curve, shape], + ] as const) { + host.fields.set(rowKey, row); + host.body.add(row.row); + } + } } export function refreshTimingRows(host: InspectorHost, holderId: string, objectId: string): void { @@ -271,6 +329,11 @@ export function refreshTimingRows(host: InspectorHost, holderId: string, objectI host.fields.get(START_ROW)?.setValue(found.item.start / fps); host.fields.get(LENGTH_ROW)?.setValue(found.item.length / fps); host.fields.get(OFFSET_ROW)?.setValue(found.item.offset); + const fades = itemFades(found.item); + host.fields.get(FADE_ROWS.in.key)?.setValue(fades.fadeIn / fps); + host.fields.get(FADE_ROWS.out.key)?.setValue(fades.fadeOut / fps); + host.fields.get(FADE_ROWS.in.curve)?.setValue(fades.curveIn); + host.fields.get(FADE_ROWS.out.curve)?.setValue(fades.curveOut); } /** What shapes the timing rows: whether there is a video, and whether the length still follows it. */ diff --git a/src/runtime/systems/CompositionSystem.ts b/src/runtime/systems/CompositionSystem.ts index e66b9551..72332b90 100644 --- a/src/runtime/systems/CompositionSystem.ts +++ b/src/runtime/systems/CompositionSystem.ts @@ -14,7 +14,13 @@ import { contentId, placedWithContent } from '../../core/model/instance'; import { findAssetOf } from '../../core/model/Project'; import type { PropertyValue } from '../../core/model/Properties'; import { findVideo, VIDEO_WAIT_MS } from '../../core/model/video'; -import { VIDEO_CLOCK_KEY, VIDEO_TIME_KEY, videoDrivenOf } from '../../core/properties/videos'; +import { + VIDEO_CLOCK_KEY, + VIDEO_LEVEL_KEY, + VIDEO_TIME_KEY, + videoDrivenOf, + videoLevelOf, +} from '../../core/properties/videos'; import { CompositionSound } from '../compositionSound'; import type { Entity } from '../Entity'; import type { World } from '../World'; @@ -29,11 +35,14 @@ export interface CompositionFinish { startedBy: readonly object[]; } -/** An item's object in the run, with what its clips started from and the sprites whose video it drives. */ +/** + * An item's object in the run, with what its clips started from and the sprites whose video it + * drives, each with the Video volume it came with: the item's fades scale that. + */ interface Spawned { entity: Entity; bases: Map; - videos: Entity[]; + videos: { sprite: Entity; level: number }[]; } interface Run { @@ -142,7 +151,7 @@ export class CompositionSystem implements System { const pictures = this.pictures; const missing = pictures !== null && - [...run.live.values()].some(({ videos }) => videos.some((v) => pictures.missing(v.objectId))); + [...run.live.values()].some(({ videos }) => videos.some((v) => pictures.missing(v.sprite.objectId))); if (!missing) { run.waited = 0; return false; @@ -192,16 +201,19 @@ function spawn(world: World, placement: Entity, item: LiveItem): Spawned | null const entity = entities.find((e) => e.objectId === id); if (!entity) return null; // Below the item, a sprite set to be driven by hand keeps its own time. - const videos = entities.filter((e) => e.type === 'sprite' && (e === entity || !videoDrivenOf(e.props))); + const videos = entities + .filter((e) => e.type === 'sprite' && (e === entity || !videoDrivenOf(e.props))) + .map((sprite) => ({ sprite, level: videoLevelOf(sprite.props) })); return { entity, bases: new Map(), videos }; } function play(world: World, spawned: Spawned, item: LiveItem, curves: ReturnType): void { - for (const sprite of spawned.videos) { + for (const { sprite, level } of spawned.videos) { const texture = sprite.props.texture; if (typeof texture !== 'string' || !findVideo(world.project, texture)) continue; sprite.props[VIDEO_CLOCK_KEY] = 'driven'; sprite.props[VIDEO_TIME_KEY] = item.videoTime; + sprite.props[VIDEO_LEVEL_KEY] = level * item.level; } for (const clip of item.clips) { for (const b of bind(world, clip.animation, spawned.entity)) { diff --git a/src/viewport/SoundLibrary.ts b/src/viewport/SoundLibrary.ts index a12309e2..10d421cc 100644 --- a/src/viewport/SoundLibrary.ts +++ b/src/viewport/SoundLibrary.ts @@ -30,12 +30,12 @@ import { arrivalOf, sendDelays, type TimedBus } from '../core/music/routing'; import type { SendTap } from '../core/graph/Graph'; import { MASTER_MIX, type MixBus, type MixSend, sameMix } from '../core/music/mixer'; import { limiterLatency } from '../core/soundfx/dynamics'; -import type { ScheduledClip, ScheduledNote } from '../core/music/Scheduler'; +import type { ScheduledClip, ScheduledNote, ScheduledRamp } from '../core/music/Scheduler'; import { dbToGain, SOUND_ITEM_MAX_DB } from '../core/music/song'; import { DEFAULT_TEMPO } from '../core/music/time'; import { playedPlan, type StreamedFile, type StreamPlan } from '../core/music/streams'; import { STREAM_RATE } from '../core/music/streamQueue'; -import { type CutRange, cutChannels, cutRange, parseCutName } from '../core/model/sound'; +import { type CutRange, cutChannels, cutRange, fadeGain, parseCutName } from '../core/model/sound'; import { shifterLatency } from '../core/soundfx/pitchPlayer'; import { REVERB_MAX_DECAY } from '../core/soundfx/reverb'; import type { AudioSink, ClipTiming, NoteTiming } from '../runtime/systems/Audio'; @@ -94,6 +94,8 @@ const GLIDE_MEMORY = 2; const PEAK_BUCKETS = 1024; /** Samples the cuts made so far may hold together before the oldest are let go; one is always kept. */ const CUT_BUDGET = 64 * 1024 * 1024; +/** Points a ramp of a bus's level is drawn with (#418); the audio clock glides between them. */ +const RAMP_POINTS = 64; /** * One way's sound as a voice holds it: a buffer source, or the pitch worklet where the way has a @@ -592,10 +594,14 @@ export class SoundLibrary implements AudioSink { return { latency: bus.chain?.latency ?? 0, into: bus.into, ...(bus.sends && { sends: bus.sends }) }; }; - /** What each track's bus was last set to, with whether its sound FX is there and how late (test hook). */ + /** + * What each track's bus was last set to, with whether its sound FX is there and how late, and the + * level it has now, ramps and fades included (test hook). + */ busLevels(): { name: string; gain: number; + level: number; pan: number; into: string | null; fx: boolean; @@ -605,6 +611,7 @@ export class SoundLibrary implements AudioSink { return [...this.buses].map(([name, bus]) => ({ name, ...bus.set, + level: bus.gain.gain.value, into: bus.sends ? null : bus.into, fx: bus.chain !== null, latency: bus.chain?.latency ?? 0, @@ -1131,6 +1138,10 @@ export class SoundLibrary implements AudioSink { ...(strikeClip && { strikeClip: (clip: ScheduledClip) => strikeClip({ ...clip, bus: into(clip.bus) }), }), + // Only a track's: the master of the playing is its level and its fades. + ramp: (ramp) => { + if (tracks.has(ramp.bus)) this.rampBus(groupBus(id, ramp.bus), ramp); + }, }; this.sequences.start(id, name, routed, owner); return id; @@ -1382,6 +1393,31 @@ export class SoundLibrary implements AudioSink { return Math.max(0, ...this.groupBuses(id).map((name) => this.busLatency(name))); } + /** + * A track of a playing ramped (#418): set to its gain where the ramp takes no time, else along its + * curve from where the ramp was taken up. + */ + private rampBus(name: string, ramp: ScheduledRamp): void { + const param = this.buses.get(name)?.gain.gain; + if (!param) return; + const at = Math.max(this.audio().currentTime, ramp.at); + const end = ramp.at + ramp.duration; + const level = (t: number) => ramp.gain * fadeGain(ramp.curve, ramp.rise ? t : 1 - t); + param.cancelScheduledValues(at); + if (!(end > at)) { + param.setValueAtTime(level(1), at); + return; + } + const values = Float32Array.from({ length: RAMP_POINTS }, (_, i) => + level(ramp.done + ((1 - ramp.done) * i) / (RAMP_POINTS - 1)), + ); + try { + param.setValueCurveAtTime(values, at, end - at); + } catch { + param.setValueAtTime(level(1), end); + } + } + /** The master of a playing, from where it stands at `at` to `gain`, there `seconds` later. */ private level(id: number, gain: number, at: number, seconds: number): void { const master = this.buses.get(groupBus(id, '')); diff --git a/src/viewport/VideoLibrary.ts b/src/viewport/VideoLibrary.ts index 88c34e10..804113ff 100644 --- a/src/viewport/VideoLibrary.ts +++ b/src/viewport/VideoLibrary.ts @@ -22,7 +22,7 @@ import { parseVideoKey, type VideoPlay, } from '../core/model/video'; -import { videoDrivenOf, videoTimeOf } from '../core/properties/videos'; +import { videoDrivenOf, videoLevelOf, videoTimeOf } from '../core/properties/videos'; import { FileLibrary } from './FileLibrary'; import { soundLibrary } from './SoundLibrary'; import { followVideo, readSpeed, type SpeedReading } from './videoFollow'; @@ -44,6 +44,8 @@ const IDLE_MS = 2000; * otherwise, heard, until it is let go. */ const HUSH_MS = 100; +/** Seconds a driven video's level takes to get most of the way to a new one, see `hear`. */ +const LEVEL_GLIDE = 0.015; /** * When the frame being drawn began, in milliseconds like `performance.now()`. The ticker moves every @@ -69,6 +71,8 @@ interface Drive extends SeekChase { time: number; /** Whether and how the wearer lets it be heard, when it plays at about 1x. */ heard: VideoHearing; + /** How loud the wearer hears it, 0–1, on top of the video's own volume: its Video volume. */ + level: number; reading: SpeedReading | null; } @@ -79,6 +83,8 @@ interface Player { texture: Texture | null; /** The sound on its way into its bus, made the first time a run lets it be heard. */ gain: GainNode | null; + /** The level its gain was last sent to. */ + level: number; gone: boolean; /** When a sprite last asked for its picture, by `performance.now()`. */ asked: number; @@ -134,6 +140,7 @@ export class VideoLibrary { const drive = player.drive; drive.time = time; drive.heard = heard; + drive.level = videoLevelOf(props); drive.reading = readSpeed(drive.reading, time, frameTime()); if (player.texture) this.follow(player, drive); return { ...ref, file: key }; @@ -290,7 +297,7 @@ export class VideoLibrary { const play = parseVideoKey(key); if (!play) return null; const drive: Drive | null = isDrivenVideoKey(key) - ? { time, heard: 'none', reading: null, wanted: null, landing: false } + ? { time, heard: 'none', level: 1, reading: null, wanted: null, landing: false } : null; const at = drive ? drivenTimeOf(play, time, NaN) : play.start; const player: Player = { @@ -305,6 +312,7 @@ export class VideoLibrary { }), texture: null, gain: null, + level: -1, gone: false, asked: performance.now(), drive, @@ -351,10 +359,18 @@ export class VideoLibrary { element.requestVideoFrameCallback(check); } - /** Along the master routing of its video where `routed`, else straight into the master. */ + /** + * Along the master routing of its video where `routed`, else straight into the master. A driven + * one's level glides to what its wearer asks: set once a frame, a fade stepped and clicked. + */ private hear(player: Player, routed = true): void { + const made = player.gain === null; player.gain ??= soundLibrary.mediaInput(player.element, player.play.file, routed); - if (player.gain) player.gain.gain.value = player.play.volume; + const level = player.play.volume * (player.drive?.level ?? 1); + if (player.gain && made) player.gain.gain.value = level; + else if (player.gain && player.level !== level) + player.gain.gain.setTargetAtTime(level, player.gain.context.currentTime, LEVEL_GLIDE); + player.level = level; player.element.muted = false; } diff --git a/tests/core/commands.test.ts b/tests/core/commands.test.ts index 2cbe05cd..ccd82ac4 100644 --- a/tests/core/commands.test.ts +++ b/tests/core/commands.test.ts @@ -37,6 +37,7 @@ describe('asset commands', () => { texture: '', videoClock: 'free', videoTime: 0, + videoVolume: 1, shader: '', }); }); diff --git a/tests/core/composition-audio.test.ts b/tests/core/composition-audio.test.ts index 7cd0ca0b..7fd0db4c 100644 --- a/tests/core/composition-audio.test.ts +++ b/tests/core/composition-audio.test.ts @@ -19,7 +19,12 @@ import type { InstrumentAsset, SongAsset, } from '../../src/core/model/Asset'; -import { compositionLength, fittedLength, setSoundMeasure } from '../../src/core/model/composition'; +import { + compositionLength, + fittedLength, + itemFades, + setSoundMeasure, +} from '../../src/core/model/composition'; import { carriedClips, itemMedia, mediaMarks } from '../../src/core/model/compositionEdit'; import { findAssetOf } from '../../src/core/model/Project'; import { audioItemBus, compositionPlayings, compositionSpan } from '../../src/core/music/compositionAudio'; @@ -202,6 +207,44 @@ describe('the sequences of a composition', () => { ]); }); + it('fade an item on its bus: up from silence – at once without a fade in –, down at its end (#418)', () => { + const f = makeCut(); + f.lay( + { ...f.audioItem('a', 'sound', 5, 20, { volume: -6 }), fadeIn: 5, fadeOut: 10, curveOut: 'sCurve' }, + { ...f.audioItem('b', 'pattern', 30, 10), fadeOut: 4 }, + f.audioItem('c', 'sound', 50, 10), + ); + const [playing] = compositionPlayings(f.project, f.cut()); + const bus = (id: string) => audioItemBus(f.cut().id, id); + const levels = playing!.sequence.levels!.map((l) => ({ ...l, gain: Number(l.gain.toFixed(3)) })); + expect(levels).toEqual([ + { tick: 500, length: 500, bus: bus('a'), gain: 0.501, rise: true, curve: 'linear' }, + { tick: 1500, length: 1000, bus: bus('a'), gain: 0.501, rise: false, curve: 'sCurve' }, + { tick: 3000, length: 0, bus: bus('b'), gain: 1, rise: true, curve: 'linear' }, + { tick: 3600, length: 400, bus: bus('b'), gain: 1, rise: false, curve: 'linear' }, + ]); + }); + + it('keep fades in whole frames above 0 and curves there are, and work them no longer than the item', () => { + const f = makeCut(); + const raw = { fadeIn: 2.6, fadeOut: -3, curveIn: 'bogus', curveOut: 'equalPower' }; + f.lay({ ...f.audioItem('a', 'sound', 0, 10), ...raw } as unknown as CompositionItem); + const [held] = f.cut().items; + expect([held!.fadeIn, held!.fadeOut, held!.curveIn, held!.curveOut]).toEqual([ + 3, + undefined, + undefined, + 'equalPower', + ]); + expect(itemFades({ ...held!, fadeIn: 50, fadeOut: 4 })).toEqual({ + fadeIn: 10, + fadeOut: 0, + curveIn: 'linear', + curveOut: 'equalPower', + }); + expect(itemFades({ ...held!, fadeIn: 6, fadeOut: 9 }).fadeOut).toBe(4); + }); + it('play a pattern round and round at its own tempo, cut where the item ends', () => { const f = makeCut(); // 3.5 s from half a second in: beat 3 (1 s), then round two (1.5 s, 2.5 s), then the next (3.5 s) is cut. diff --git a/tests/core/composition-edit.test.ts b/tests/core/composition-edit.test.ts index 71c05bc6..0e2213a3 100644 --- a/tests/core/composition-edit.test.ts +++ b/tests/core/composition-edit.test.ts @@ -21,6 +21,7 @@ import { clampedMove, droppedLength, droppedSpan, + fadedItem, heldMove, type ItemMedia, mediaEdges, @@ -43,6 +44,7 @@ import { overlaps, uncoveredEnd, } from '../../src/editor/composition/compositionView'; +import { fadeLevel, itemFades } from '../../src/core/model/composition'; import { placementRejection } from '../../src/core/model/placement'; import { findAsset, findAssetOf } from '../../src/core/model/Project'; import { makeProject } from '../helpers'; @@ -517,3 +519,44 @@ describe('an item that follows its media (fitMedia)', () => { expect(cut.comp().items[0]!.length).toBe(90); }); }); + +describe('the fades of an item (#418)', () => { + const sound = item('a', 't1', 100, 60, { fadeIn: 10, fadeOut: 20 }); + + it('go where their handle is dragged, no further than the other fade, and none at the edge', () => { + expect(fadedItem(sound, 'in', 130).fadeIn).toBe(30); + expect(fadedItem(sound, 'in', 150).fadeIn).toBe(40); + expect('fadeIn' in fadedItem(sound, 'in', 90)).toBe(false); + expect(fadedItem(sound, 'out', 150.4).fadeOut).toBe(10); + expect(fadedItem(sound, 'out', 100).fadeOut).toBe(50); + expect('fadeOut' in fadedItem(sound, 'out', 170)).toBe(false); + }); + + it('shape the level along their curves, silent at the very edges', () => { + expect(fadeLevel(sound, 0)).toBe(0); + expect(fadeLevel(sound, 5)).toBeCloseTo(0.5, 9); + expect(fadeLevel(sound, 30)).toBe(1); + expect(fadeLevel(sound, 50)).toBeCloseTo(0.5, 9); + expect(fadeLevel({ ...sound, curveIn: 'equalPower' }, 5)).toBeCloseTo(Math.SQRT1_2, 9); + expect(fadeLevel(item('b', 't1', 0, 60), 0)).toBe(1); + }); + + it('are taken by their handles on the top edge, before the item’s ends', () => { + const view = { panX: 0, pxPerTick: 2, panY: 0, trackHeight: 48 }; + const tracks = [{ id: 't1', name: 'Track 1' }]; + const fades = (i: CompositionItem) => (i.id === 'a' ? itemFades(i) : null); + const hit = (x: number, y: number, list = [sound]) => + hitAt(view, tracks, list, [], () => 0, { x, y }, fades); + // The item runs from x 200 to 320; its fade in ends at 220, its fade out starts at 280. + expect(hit(222, 4)).toEqual({ kind: 'item', id: 'a', part: 'fadeIn' }); + expect(hit(278, 10)).toEqual({ kind: 'item', id: 'a', part: 'fadeOut' }); + expect(hit(222, 30)).toEqual({ kind: 'item', id: 'a', part: 'body' }); + expect(hit(250, 4)).toEqual({ kind: 'item', id: 'a', part: 'body' }); + // Without fades the handles sit on the corners; lower down the corner is the edge. + const plain = item('a', 't1', 100, 60); + expect(hit(202, 4, [plain])).toEqual({ kind: 'item', id: 'a', part: 'fadeIn' }); + expect(hit(202, 30, [plain])).toEqual({ kind: 'item', id: 'a', part: 'start' }); + // An item without sound has none. + expect(hit(202, 4, [item('b', 't1', 100, 60)])).toEqual({ kind: 'item', id: 'b', part: 'start' }); + }); +}); diff --git a/tests/core/composition-play.test.ts b/tests/core/composition-play.test.ts index 3c7ec45e..787821cc 100644 --- a/tests/core/composition-play.test.ts +++ b/tests/core/composition-play.test.ts @@ -17,9 +17,10 @@ import { stopComposition, } from '../../src/core/model/composition'; import { contentId, instanceChildId, resolveObjects } from '../../src/core/model/instance'; -import { findAssetOf, findScene } from '../../src/core/model/Project'; +import { findAsset, findAssetOf, findScene } from '../../src/core/model/Project'; import { createSceneObject } from '../../src/core/model/SceneObject'; -import { VIDEO_CLOCK_KEY, VIDEO_TIME_KEY } from '../../src/core/properties/videos'; +import { VIDEO_CLOCK_KEY, VIDEO_LEVEL_KEY, VIDEO_TIME_KEY } from '../../src/core/properties/videos'; +import { resolveSceneObjectProps } from '../../src/core/properties/resolve'; import { addGameSystems } from '../../src/runtime/systems/BehaviorSystem'; import type { VideoPictures } from '../../src/runtime/systems/CompositionSystem'; import { Input } from '../../src/runtime/systems/Input'; @@ -259,6 +260,40 @@ describe('a composition played in a scene', () => { expect(run.at(item)!.props[VIDEO_TIME_KEY]).toBeCloseTo(2, 9); }); + it('fades the video of an item: its Video volume scaled by the fades, in the game and in the tab (#418)', () => { + const stage = makeStage(); + director(stage); + const video = new CreateVideoCommand('Intro', FILE, { + width: 320, + height: 180, + duration: 10, + fps: 30, + audio: true, + }); + stage.stack.execute(video); + const screen = new CreateAssetCommand('sprite', 'Screen', { + texture: video.asset.id, + [VIDEO_LEVEL_KEY]: 0.8, + }); + stage.stack.execute(screen); + const item = stage.item(first(stage), screen.asset.id, 0, 50); + Object.assign(stage.comp().items[0]!, { fadeIn: 10, fadeOut: 20 }); + const run = play(stage); + run.frame(6); + expect(run.at(item)!.props[VIDEO_LEVEL_KEY]).toBeCloseTo(0.4, 9); + run.frame(30); + expect(run.at(item)!.props[VIDEO_LEVEL_KEY]).toBeCloseTo(0.6, 9); + const heard = (frame: number) => { + const holder = compositionHolder(stage.project, stage.comp(), frame); + const object = holder.objects.find((o) => o.id === item)!; + return resolveSceneObjectProps(stage.project, findAsset(stage.project, object.assetId)!, object)[ + VIDEO_LEVEL_KEY + ]; + }; + expect(heard(5)).toBeCloseTo(0.4, 9); + expect(heard(20)).toBe(0.8); + }); + it('stands still while the view waits for the first picture of an item’s video, 3 s at most', () => { const stage = makeStage(); director(stage); diff --git a/tests/core/music.test.ts b/tests/core/music.test.ts index fdd41bca..277e4190 100644 --- a/tests/core/music.test.ts +++ b/tests/core/music.test.ts @@ -4,11 +4,13 @@ import { LEAD, LOOKAHEAD, type ScheduledNote, + type ScheduledRamp, Scheduler, type SchedulerHost, } from '../../src/core/music/Scheduler'; import { heldLoop, + levelsIn, notesIn, playheadAt, type Sequence, @@ -324,3 +326,34 @@ describe('the scheduler on the audio clock (#332)', () => { expect(host.played.at(-1)!.tempo).toBe(60); }); }); + +describe('the levels of a sequence (#418)', () => { + const level = { tick: Q, length: Q, bus: 'item', gain: 0.5, rise: true, curve: 'linear' } as const; + const faded: Sequence = { notes: [note(0)], levels: [level] }; + + it('are found like clips: one the playhead lands in is taken up there', () => { + expect(levelsIn(faded, 0, null, 0, 4 * Q, true)).toEqual([{ ...level, at: Q, skip: 0 }]); + expect(levelsIn(faded, Q + Q / 4, null, 0, Q, true)).toEqual([ + { ...level, at: 0, length: (3 * Q) / 4, skip: Q / 4 }, + ]); + expect(levelsIn(faded, Q + Q / 4, null, 0, Q, false)).toEqual([]); + }); + + it('are handed to the player as ramps on the audio clock, from where the playhead took them up', () => { + const { sink, run } = fakeHost(); + const ramps: ScheduledRamp[] = []; + const scheduler = new Scheduler({ ...sink, ramp: (r) => void ramps.push(r) }, () => faded, { + tempo: 120, + }); + scheduler.start(); + run(1); + expect(ramps).toHaveLength(1); + expect(ramps[0]).toMatchObject({ bus: 'item', gain: 0.5, rise: true, curve: 'linear', done: 0 }); + expect(ramps[0]!.at).toBeCloseTo(10 + LEAD + 0.5, 9); + expect(ramps[0]!.duration).toBeCloseTo(0.5, 9); + scheduler.start(Q + Q / 2); + expect(ramps).toHaveLength(2); + expect(ramps[1]!.done).toBeCloseTo(0.5, 9); + expect(ramps[1]!.duration).toBeCloseTo(0.25, 9); + }); +}); diff --git a/tests/core/resolve.test.ts b/tests/core/resolve.test.ts index 9add0cbd..ff83bd40 100644 --- a/tests/core/resolve.test.ts +++ b/tests/core/resolve.test.ts @@ -38,6 +38,7 @@ describe('PropertySchema', () => { texture: '', videoClock: 'free', videoTime: 0, + videoVolume: 1, shader: '', }); }); @@ -73,6 +74,7 @@ describe('resolve', () => { texture: '', videoClock: 'free', videoTime: 0, + videoVolume: 1, shader: '', }); }); @@ -96,6 +98,7 @@ describe('resolve', () => { texture: '', videoClock: 'free', videoTime: 0, + videoVolume: 1, shader: '', }); diff --git a/tests/core/serialize.test.ts b/tests/core/serialize.test.ts index dab056c6..10fc8046 100644 --- a/tests/core/serialize.test.ts +++ b/tests/core/serialize.test.ts @@ -185,6 +185,7 @@ describe('serialize', () => { texture: '', videoClock: 'free', videoTime: 0, + videoVolume: 1, shader: '', }); }); diff --git a/tests/core/shaders.test.ts b/tests/core/shaders.test.ts index 62a4977e..fe9b18a2 100644 --- a/tests/core/shaders.test.ts +++ b/tests/core/shaders.test.ts @@ -729,6 +729,7 @@ describe('a sprite wearing a shader', () => { 'texture', 'videoClock', 'videoTime', + 'videoVolume', 'shader', ]); expect(schemaFor('sprite').find((d) => d.key === SHADER_KEY)).toMatchObject({ @@ -752,6 +753,7 @@ describe('a sprite wearing a shader', () => { 'texture', 'videoClock', 'videoTime', + 'videoVolume', 'shader', 'brightness', 'contrast', @@ -760,14 +762,14 @@ describe('a sprite wearing a shader', () => { expect(assetDeviates(f.project, f.sprite())).toBe(true); // A dangling id adds no rows and draws nothing – the picker still shows it. f.sprite().props[SHADER_KEY] = 'gone'; - expect(keys()).toHaveLength(15); + expect(keys()).toHaveLength(16); }); it('gives a placed object the rows of the shader it overrides, filled with the node defaults', () => { const f = wearing(); const id = place(f, f.spriteId, { x: 0, y: 0 }); const object = () => f.object(id); - expect(schemaForObject(f.project, f.sprite(), object())).toHaveLength(15); + expect(schemaForObject(f.project, f.sprite(), object())).toHaveLength(16); f.stack.execute(new SetOverrideCommand(f.sceneId, id, SHADER_KEY, f.s.id)); expect(schemaForObject(f.project, f.sprite(), object()).map((d) => d.key)).toContain('saturation'); expect(resolveSceneObjectProps(f.project, f.sprite(), object())).toMatchObject({ diff --git a/tests/core/texture-commands.test.ts b/tests/core/texture-commands.test.ts index 053c0553..d1fef406 100644 --- a/tests/core/texture-commands.test.ts +++ b/tests/core/texture-commands.test.ts @@ -76,6 +76,7 @@ describe('CreateAssetCommand with starting props', () => { texture: texture.asset.id, videoClock: 'free', videoTime: 0, + videoVolume: 1, shader: '', }); stack.undo(); diff --git a/tests/viewport/sound-library.test.ts b/tests/viewport/sound-library.test.ts index ae7f8e18..93fa7475 100644 --- a/tests/viewport/sound-library.test.ts +++ b/tests/viewport/sound-library.test.ts @@ -125,7 +125,7 @@ let moduleLoaded = false; */ const INTO_MASTER = ['gain > gain', 'gain > panner', 'panner > destination', 'gain > gain']; /** What `busLevels` says of that master while nothing has set it. */ -const MASTER_LEVEL = { name: 'mix', gain: 1, pan: 0, into: null, fx: false, latency: 0 }; +const MASTER_LEVEL = { name: 'mix', gain: 1, pan: 0, into: null, fx: false, latency: 0, level: 1 }; const sources: FakeSource[] = []; /** Every `connect` made, as "from > to" by the kind of node – the order a voice is wired in. */ @@ -1900,8 +1900,16 @@ describe('the patterns and songs a game plays (#336)', () => { // The master of the project is made on the way, the one all of it is heard through (#325). expect(library.busLevels()).toEqual([ MASTER_LEVEL, - { name: `play:${id}`, gain: 0, pan: 0, into: 'mix', fx: false, latency: 0 }, - { name: `play:${id}:song:s:t`, gain: 0.25, pan: -1, into: `play:${id}`, fx: false, latency: 0 }, + { name: `play:${id}`, gain: 0, pan: 0, into: 'mix', fx: false, latency: 0, level: 0 }, + { + name: `play:${id}:song:s:t`, + gain: 0.25, + pan: -1, + into: `play:${id}`, + fx: false, + latency: 0, + level: 0.25, + }, ]); expect(wires).toEqual([ 'gain > gain', @@ -2350,7 +2358,7 @@ describe('a sound FX on a bus (#346)', () => { await library.setBuses([{ ...track, fx: undefined }]); expect(library.busLevels()).toEqual([ MASTER_LEVEL, - { name: 'song:s:t', gain: 1, pan: 0, into: 'mix', fx: false, latency: 0 }, + { name: 'song:s:t', gain: 1, pan: 0, into: 'mix', fx: false, latency: 0, level: 1 }, ]); }); @@ -2469,6 +2477,7 @@ describe('the sends of the tracks of a song (#350)', () => { into: 'mix', fx: false, latency: 0, + level: 1, }); }); diff --git a/tools/e2e/69-composition-tab.mjs b/tools/e2e/69-composition-tab.mjs index 2d42ced2..f593fda0 100644 --- a/tools/e2e/69-composition-tab.mjs +++ b/tools/e2e/69-composition-tab.mjs @@ -8,7 +8,8 @@ * whose player is still loading. A copied item is pasted under the pointer or at the playhead. A * video dragged in wears a sprite the composition keeps to itself. An item dragged in the picture * follows the pointer while the button is down. A band picks items, with Alt clips, and copied clips - * are pasted where they fit. A sound dragged in is an audio item, heard while the playhead runs. + * are pasted where they fit. A sound dragged in is an audio item, heard while the playhead runs; + * the handle of its fade in drags out a fade its bus rises along. * * Everything it adds carries a `ct69` id and goes again at the end. */ @@ -591,6 +592,48 @@ export async function compositionTab(t) { JSON.stringify({ patternDrop, songDrop, kinds }), ); + // 69p. The handle on the top left corner of a sound item lights up under the pointer and drags + // its fade in out to 10 frames; played from the item's start, its bus rises from silence over that + // second and stays up. + await mutate(` + __gm.state.project.assets.find((a) => a.id === '${CUT}').items = [ + { id: 'ct69-fade', track: 'ct69-t2', start: 10, length: 40, offset: 0, + audio: { kind: 'sound', asset: 'ct69-sound', volume: 0, bus: '' } }, + ];`); + await evaluate('__gm.composition.scrub(0)'); + await sleep(100); + const fadeBox = await json(`__gm.composition.box('ct69-fade')`); + const perFrame = (await json('__gm.composition.view()')).pxPerTick; + await mouse('mouseMoved', fadeBox.x + 1, fadeBox.y + 3); + await sleep(50); + const fadeHover = (await json('__gm.composition.view()')).hover; + await drag(fadeBox.x + 1, fadeBox.y + 3, fadeBox.x + 10 * perFrame, fadeBox.y + 3); + const faded = (await model()).items[0]; + await evaluate('__gm.composition.scrub(10)'); + const fadeAt = await json('__gm.composition.playAt()'); + await t.click(fadeAt.x, fadeAt.y); + const levels = []; + for (let i = 0; i < 16; i++) { + await sleep(100); + const bus = (await json('__gm.song.buses()')).find((b) => + b.name.endsWith(`composition:${CUT}:ct69-fade`), + ); + levels.push(bus ? Number(bus.level.toFixed(3)) : null); + } + await t.click(fadeAt.x, fadeAt.y); + const heardLevels = levels.filter((l) => l !== null); + check( + 'the fade handle of a sound item drags its fade in, and its bus rises from silence over it', + fadeHover?.id === 'ct69-fade' && + fadeHover.part === 'fadeIn' && + faded?.fadeIn === 10 && + faded.fadeOut === undefined && + heardLevels[0] < 0.5 && + heardLevels.some((l) => l > 0.1 && l < 0.9) && + heardLevels.at(-1) > 0.99, + JSON.stringify({ fadeBox, perFrame, fadeHover, faded, levels }), + ); + await showTab('scene'); await mutate(` const p = __gm.state.project;