Fade sounds and videos in and out on the items of a composition

A trailer needs its music and its clips to come in softly and leave
softly. Until now an item of a composition could only play at one
level, and a video item could not even be turned down on its own.

- Every item with sound gets a grip at its upper left and upper right
  corner. Dragging one inwards fades the item in or out. Dragging it
  back to the edge removes the fade. Lower down the same edge still
  trims as before.
- The fade is drawn as its curve, with the part above it darkened.
  The inspector shows Fade in and Fade out in seconds, each with the
  curves sound clips already know.
- The model keeps fadeIn and fadeOut in frames plus curveIn and
  curveOut on the item. Old projects load unchanged. An item shorter
  than its fades plays them cut to its length but keeps the setting.
- Sound, pattern and song items ramp the gain of their own bus. The
  scheduler plans the ramps on the audio clock next to the notes.
  Starting in the middle of a fade picks it up from there.
- Video items set the new sprite property Video volume each frame.
  It multiplies with the fade and is only shown for a driven video.
  The player glides to each new level over 15 ms so it does not click
  between frames.

Not covered: a container item that carries a video further down gets
no grips. Only a video on the item's own sprite counts.

Tests:
- tsc, eslint, Prettier clean.
- Unit tests for composition, music, song and sound (12 files): 269
  passed, 7 of them new.
- e2e 69 on melektaus-atx: ALL PASSED, 21 checks. 69p drags a grip to
  a fade in of 10 frames and hears the item bus rise from 0.127 over
  0.229 and 0.339 to 1 within one second.
- Counter-checks as unit tests: without the audio ramps, and without
  the level on the video sprite in the game, both go red.
- Gate on melektaus-atx: 309 files, 4796 tests passed, 8 more than
  at 77ac9d1. The first run failed 14 tests in 6 files that list the
  default sprite properties or the bus levels in full. They now
  expect videoVolume and the new level field of busLevels.
- Not tried in a browser: video fades, pattern and song items with a
  fade.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/commands.test.ts

@@ -37,6 +37,7 @@ describe('asset commands', () => {
       texture: '',
       videoClock: 'free',
       videoTime: 0,
+      videoVolume: 1,
       shader: '',
     });
   });

tests/core/composition-audio.test.ts

@@ -19,7 +19,12 @@ import type {
   InstrumentAsset,
   SongAsset,
 } from '../../src/core/model/Asset';
-import { compositionLength, fittedLength, setSoundMeasure } from '../../src/core/model/composition';
+import {
+  compositionLength,
+  fittedLength,
+  itemFades,
+  setSoundMeasure,
+} from '../../src/core/model/composition';
 import { carriedClips, itemMedia, mediaMarks } from '../../src/core/model/compositionEdit';
 import { findAssetOf } from '../../src/core/model/Project';
 import { audioItemBus, compositionPlayings, compositionSpan } from '../../src/core/music/compositionAudio';
@@ -202,6 +207,44 @@ describe('the sequences of a composition', () => {
     ]);
   });
 
+  it('fade an item on its bus: up from silence – at once without a fade in –, down at its end (#418)', () => {
+    const f = makeCut();
+    f.lay(
+      { ...f.audioItem('a', 'sound', 5, 20, { volume: -6 }), fadeIn: 5, fadeOut: 10, curveOut: 'sCurve' },
+      { ...f.audioItem('b', 'pattern', 30, 10), fadeOut: 4 },
+      f.audioItem('c', 'sound', 50, 10),
+    );
+    const [playing] = compositionPlayings(f.project, f.cut());
+    const bus = (id: string) => audioItemBus(f.cut().id, id);
+    const levels = playing!.sequence.levels!.map((l) => ({ ...l, gain: Number(l.gain.toFixed(3)) }));
+    expect(levels).toEqual([
+      { tick: 500, length: 500, bus: bus('a'), gain: 0.501, rise: true, curve: 'linear' },
+      { tick: 1500, length: 1000, bus: bus('a'), gain: 0.501, rise: false, curve: 'sCurve' },
+      { tick: 3000, length: 0, bus: bus('b'), gain: 1, rise: true, curve: 'linear' },
+      { tick: 3600, length: 400, bus: bus('b'), gain: 1, rise: false, curve: 'linear' },
+    ]);
+  });
+
+  it('keep fades in whole frames above 0 and curves there are, and work them no longer than the item', () => {
+    const f = makeCut();
+    const raw = { fadeIn: 2.6, fadeOut: -3, curveIn: 'bogus', curveOut: 'equalPower' };
+    f.lay({ ...f.audioItem('a', 'sound', 0, 10), ...raw } as unknown as CompositionItem);
+    const [held] = f.cut().items;
+    expect([held!.fadeIn, held!.fadeOut, held!.curveIn, held!.curveOut]).toEqual([
+      3,
+      undefined,
+      undefined,
+      'equalPower',
+    ]);
+    expect(itemFades({ ...held!, fadeIn: 50, fadeOut: 4 })).toEqual({
+      fadeIn: 10,
+      fadeOut: 0,
+      curveIn: 'linear',
+      curveOut: 'equalPower',
+    });
+    expect(itemFades({ ...held!, fadeIn: 6, fadeOut: 9 }).fadeOut).toBe(4);
+  });
+
   it('play a pattern round and round at its own tempo, cut where the item ends', () => {
     const f = makeCut();
     // 3.5 s from half a second in: beat 3 (1 s), then round two (1.5 s, 2.5 s), then the next (3.5 s) is cut.

tests/core/composition-edit.test.ts

@@ -21,6 +21,7 @@ import {
   clampedMove,
   droppedLength,
   droppedSpan,
+  fadedItem,
   heldMove,
   type ItemMedia,
   mediaEdges,
@@ -43,6 +44,7 @@ import {
   overlaps,
   uncoveredEnd,
 } from '../../src/editor/composition/compositionView';
+import { fadeLevel, itemFades } from '../../src/core/model/composition';
 import { placementRejection } from '../../src/core/model/placement';
 import { findAsset, findAssetOf } from '../../src/core/model/Project';
 import { makeProject } from '../helpers';
@@ -517,3 +519,44 @@ describe('an item that follows its media (fitMedia)', () => {
     expect(cut.comp().items[0]!.length).toBe(90);
   });
 });
+
+describe('the fades of an item (#418)', () => {
+  const sound = item('a', 't1', 100, 60, { fadeIn: 10, fadeOut: 20 });
+
+  it('go where their handle is dragged, no further than the other fade, and none at the edge', () => {
+    expect(fadedItem(sound, 'in', 130).fadeIn).toBe(30);
+    expect(fadedItem(sound, 'in', 150).fadeIn).toBe(40);
+    expect('fadeIn' in fadedItem(sound, 'in', 90)).toBe(false);
+    expect(fadedItem(sound, 'out', 150.4).fadeOut).toBe(10);
+    expect(fadedItem(sound, 'out', 100).fadeOut).toBe(50);
+    expect('fadeOut' in fadedItem(sound, 'out', 170)).toBe(false);
+  });
+
+  it('shape the level along their curves, silent at the very edges', () => {
+    expect(fadeLevel(sound, 0)).toBe(0);
+    expect(fadeLevel(sound, 5)).toBeCloseTo(0.5, 9);
+    expect(fadeLevel(sound, 30)).toBe(1);
+    expect(fadeLevel(sound, 50)).toBeCloseTo(0.5, 9);
+    expect(fadeLevel({ ...sound, curveIn: 'equalPower' }, 5)).toBeCloseTo(Math.SQRT1_2, 9);
+    expect(fadeLevel(item('b', 't1', 0, 60), 0)).toBe(1);
+  });
+
+  it('are taken by their handles on the top edge, before the item’s ends', () => {
+    const view = { panX: 0, pxPerTick: 2, panY: 0, trackHeight: 48 };
+    const tracks = [{ id: 't1', name: 'Track 1' }];
+    const fades = (i: CompositionItem) => (i.id === 'a' ? itemFades(i) : null);
+    const hit = (x: number, y: number, list = [sound]) =>
+      hitAt(view, tracks, list, [], () => 0, { x, y }, fades);
+    // The item runs from x 200 to 320; its fade in ends at 220, its fade out starts at 280.
+    expect(hit(222, 4)).toEqual({ kind: 'item', id: 'a', part: 'fadeIn' });
+    expect(hit(278, 10)).toEqual({ kind: 'item', id: 'a', part: 'fadeOut' });
+    expect(hit(222, 30)).toEqual({ kind: 'item', id: 'a', part: 'body' });
+    expect(hit(250, 4)).toEqual({ kind: 'item', id: 'a', part: 'body' });
+    // Without fades the handles sit on the corners; lower down the corner is the edge.
+    const plain = item('a', 't1', 100, 60);
+    expect(hit(202, 4, [plain])).toEqual({ kind: 'item', id: 'a', part: 'fadeIn' });
+    expect(hit(202, 30, [plain])).toEqual({ kind: 'item', id: 'a', part: 'start' });
+    // An item without sound has none.
+    expect(hit(202, 4, [item('b', 't1', 100, 60)])).toEqual({ kind: 'item', id: 'b', part: 'start' });
+  });
+});

tests/core/composition-play.test.ts

@@ -17,9 +17,10 @@ import {
   stopComposition,
 } from '../../src/core/model/composition';
 import { contentId, instanceChildId, resolveObjects } from '../../src/core/model/instance';
-import { findAssetOf, findScene } from '../../src/core/model/Project';
+import { findAsset, findAssetOf, findScene } from '../../src/core/model/Project';
 import { createSceneObject } from '../../src/core/model/SceneObject';
-import { VIDEO_CLOCK_KEY, VIDEO_TIME_KEY } from '../../src/core/properties/videos';
+import { VIDEO_CLOCK_KEY, VIDEO_LEVEL_KEY, VIDEO_TIME_KEY } from '../../src/core/properties/videos';
+import { resolveSceneObjectProps } from '../../src/core/properties/resolve';
 import { addGameSystems } from '../../src/runtime/systems/BehaviorSystem';
 import type { VideoPictures } from '../../src/runtime/systems/CompositionSystem';
 import { Input } from '../../src/runtime/systems/Input';
@@ -259,6 +260,40 @@ describe('a composition played in a scene', () => {
     expect(run.at(item)!.props[VIDEO_TIME_KEY]).toBeCloseTo(2, 9);
   });
 
+  it('fades the video of an item: its Video volume scaled by the fades, in the game and in the tab (#418)', () => {
+    const stage = makeStage();
+    director(stage);
+    const video = new CreateVideoCommand('Intro', FILE, {
+      width: 320,
+      height: 180,
+      duration: 10,
+      fps: 30,
+      audio: true,
+    });
+    stage.stack.execute(video);
+    const screen = new CreateAssetCommand('sprite', 'Screen', {
+      texture: video.asset.id,
+      [VIDEO_LEVEL_KEY]: 0.8,
+    });
+    stage.stack.execute(screen);
+    const item = stage.item(first(stage), screen.asset.id, 0, 50);
+    Object.assign(stage.comp().items[0]!, { fadeIn: 10, fadeOut: 20 });
+    const run = play(stage);
+    run.frame(6);
+    expect(run.at(item)!.props[VIDEO_LEVEL_KEY]).toBeCloseTo(0.4, 9);
+    run.frame(30);
+    expect(run.at(item)!.props[VIDEO_LEVEL_KEY]).toBeCloseTo(0.6, 9);
+    const heard = (frame: number) => {
+      const holder = compositionHolder(stage.project, stage.comp(), frame);
+      const object = holder.objects.find((o) => o.id === item)!;
+      return resolveSceneObjectProps(stage.project, findAsset(stage.project, object.assetId)!, object)[
+        VIDEO_LEVEL_KEY
+      ];
+    };
+    expect(heard(5)).toBeCloseTo(0.4, 9);
+    expect(heard(20)).toBe(0.8);
+  });
+
   it('stands still while the view waits for the first picture of an item’s video, 3 s at most', () => {
     const stage = makeStage();
     director(stage);

tests/core/music.test.ts

@@ -4,11 +4,13 @@ import {
   LEAD,
   LOOKAHEAD,
   type ScheduledNote,
+  type ScheduledRamp,
   Scheduler,
   type SchedulerHost,
 } from '../../src/core/music/Scheduler';
 import {
   heldLoop,
+  levelsIn,
   notesIn,
   playheadAt,
   type Sequence,
@@ -324,3 +326,34 @@ describe('the scheduler on the audio clock (#332)', () => {
     expect(host.played.at(-1)!.tempo).toBe(60);
   });
 });
+
+describe('the levels of a sequence (#418)', () => {
+  const level = { tick: Q, length: Q, bus: 'item', gain: 0.5, rise: true, curve: 'linear' } as const;
+  const faded: Sequence = { notes: [note(0)], levels: [level] };
+
+  it('are found like clips: one the playhead lands in is taken up there', () => {
+    expect(levelsIn(faded, 0, null, 0, 4 * Q, true)).toEqual([{ ...level, at: Q, skip: 0 }]);
+    expect(levelsIn(faded, Q + Q / 4, null, 0, Q, true)).toEqual([
+      { ...level, at: 0, length: (3 * Q) / 4, skip: Q / 4 },
+    ]);
+    expect(levelsIn(faded, Q + Q / 4, null, 0, Q, false)).toEqual([]);
+  });
+
+  it('are handed to the player as ramps on the audio clock, from where the playhead took them up', () => {
+    const { sink, run } = fakeHost();
+    const ramps: ScheduledRamp[] = [];
+    const scheduler = new Scheduler({ ...sink, ramp: (r) => void ramps.push(r) }, () => faded, {
+      tempo: 120,
+    });
+    scheduler.start();
+    run(1);
+    expect(ramps).toHaveLength(1);
+    expect(ramps[0]).toMatchObject({ bus: 'item', gain: 0.5, rise: true, curve: 'linear', done: 0 });
+    expect(ramps[0]!.at).toBeCloseTo(10 + LEAD + 0.5, 9);
+    expect(ramps[0]!.duration).toBeCloseTo(0.5, 9);
+    scheduler.start(Q + Q / 2);
+    expect(ramps).toHaveLength(2);
+    expect(ramps[1]!.done).toBeCloseTo(0.5, 9);
+    expect(ramps[1]!.duration).toBeCloseTo(0.25, 9);
+  });
+});

tests/core/resolve.test.ts

@@ -38,6 +38,7 @@ describe('PropertySchema', () => {
       texture: '',
       videoClock: 'free',
       videoTime: 0,
+      videoVolume: 1,
       shader: '',
     });
   });
@@ -73,6 +74,7 @@ describe('resolve', () => {
       texture: '',
       videoClock: 'free',
       videoTime: 0,
+      videoVolume: 1,
       shader: '',
     });
   });
@@ -96,6 +98,7 @@ describe('resolve', () => {
       texture: '',
       videoClock: 'free',
       videoTime: 0,
+      videoVolume: 1,
       shader: '',
     });
 

tests/core/serialize.test.ts

@@ -185,6 +185,7 @@ describe('serialize', () => {
       texture: '',
       videoClock: 'free',
       videoTime: 0,
+      videoVolume: 1,
       shader: '',
     });
   });

tests/core/shaders.test.ts

@@ -729,6 +729,7 @@ describe('a sprite wearing a shader', () => {
       'texture',
       'videoClock',
       'videoTime',
+      'videoVolume',
       'shader',
     ]);
     expect(schemaFor('sprite').find((d) => d.key === SHADER_KEY)).toMatchObject({
@@ -752,6 +753,7 @@ describe('a sprite wearing a shader', () => {
       'texture',
       'videoClock',
       'videoTime',
+      'videoVolume',
       'shader',
       'brightness',
       'contrast',
@@ -760,14 +762,14 @@ describe('a sprite wearing a shader', () => {
     expect(assetDeviates(f.project, f.sprite())).toBe(true);
     // A dangling id adds no rows and draws nothing – the picker still shows it.
     f.sprite().props[SHADER_KEY] = 'gone';
-    expect(keys()).toHaveLength(15);
+    expect(keys()).toHaveLength(16);
   });
 
   it('gives a placed object the rows of the shader it overrides, filled with the node defaults', () => {
     const f = wearing();
     const id = place(f, f.spriteId, { x: 0, y: 0 });
     const object = () => f.object(id);
-    expect(schemaForObject(f.project, f.sprite(), object())).toHaveLength(15);
+    expect(schemaForObject(f.project, f.sprite(), object())).toHaveLength(16);
     f.stack.execute(new SetOverrideCommand(f.sceneId, id, SHADER_KEY, f.s.id));
     expect(schemaForObject(f.project, f.sprite(), object()).map((d) => d.key)).toContain('saturation');
     expect(resolveSceneObjectProps(f.project, f.sprite(), object())).toMatchObject({

tests/core/texture-commands.test.ts

@@ -76,6 +76,7 @@ describe('CreateAssetCommand with starting props', () => {
       texture: texture.asset.id,
       videoClock: 'free',
       videoTime: 0,
+      videoVolume: 1,
       shader: '',
     });
     stack.undo();

tests/viewport/sound-library.test.ts

@@ -125,7 +125,7 @@ let moduleLoaded = false;
  */
 const INTO_MASTER = ['gain > gain', 'gain > panner', 'panner > destination', 'gain > gain'];
 /** What `busLevels` says of that master while nothing has set it. */
-const MASTER_LEVEL = { name: 'mix', gain: 1, pan: 0, into: null, fx: false, latency: 0 };
+const MASTER_LEVEL = { name: 'mix', gain: 1, pan: 0, into: null, fx: false, latency: 0, level: 1 };
 
 const sources: FakeSource[] = [];
 /** Every `connect` made, as "from > to" by the kind of node – the order a voice is wired in. */
@@ -1900,8 +1900,16 @@ describe('the patterns and songs a game plays (#336)', () => {
     // The master of the project is made on the way, the one all of it is heard through (#325).
     expect(library.busLevels()).toEqual([
       MASTER_LEVEL,
-      { name: `play:${id}`, gain: 0, pan: 0, into: 'mix', fx: false, latency: 0 },
-      { name: `play:${id}:song:s:t`, gain: 0.25, pan: -1, into: `play:${id}`, fx: false, latency: 0 },
+      { name: `play:${id}`, gain: 0, pan: 0, into: 'mix', fx: false, latency: 0, level: 0 },
+      {
+        name: `play:${id}:song:s:t`,
+        gain: 0.25,
+        pan: -1,
+        into: `play:${id}`,
+        fx: false,
+        latency: 0,
+        level: 0.25,
+      },
     ]);
     expect(wires).toEqual([
       'gain > gain',
@@ -2350,7 +2358,7 @@ describe('a sound FX on a bus (#346)', () => {
     await library.setBuses([{ ...track, fx: undefined }]);
     expect(library.busLevels()).toEqual([
       MASTER_LEVEL,
-      { name: 'song:s:t', gain: 1, pan: 0, into: 'mix', fx: false, latency: 0 },
+      { name: 'song:s:t', gain: 1, pan: 0, into: 'mix', fx: false, latency: 0, level: 1 },
     ]);
   });
 
@@ -2469,6 +2477,7 @@ describe('the sends of the tracks of a song (#350)', () => {
       into: 'mix',
       fx: false,
       latency: 0,
+      level: 1,
     });
   });