Fade sounds and videos in and out on the items of a composition

A trailer needs its music and its clips to come in softly and leave
softly. Until now an item of a composition could only play at one
level, and a video item could not even be turned down on its own.

- Every item with sound gets a grip at its upper left and upper right
  corner. Dragging one inwards fades the item in or out. Dragging it
  back to the edge removes the fade. Lower down the same edge still
  trims as before.
- The fade is drawn as its curve, with the part above it darkened.
  The inspector shows Fade in and Fade out in seconds, each with the
  curves sound clips already know.
- The model keeps fadeIn and fadeOut in frames plus curveIn and
  curveOut on the item. Old projects load unchanged. An item shorter
  than its fades plays them cut to its length but keeps the setting.
- Sound, pattern and song items ramp the gain of their own bus. The
  scheduler plans the ramps on the audio clock next to the notes.
  Starting in the middle of a fade picks it up from there.
- Video items set the new sprite property Video volume each frame.
  It multiplies with the fade and is only shown for a driven video.
  The player glides to each new level over 15 ms so it does not click
  between frames.

Not covered: a container item that carries a video further down gets
no grips. Only a video on the item's own sprite counts.

Tests:
- tsc, eslint, Prettier clean.
- Unit tests for composition, music, song and sound (12 files): 269
  passed, 7 of them new.
- e2e 69 on melektaus-atx: ALL PASSED, 21 checks. 69p drags a grip to
  a fade in of 10 frames and hears the item bus rise from 0.127 over
  0.229 and 0.339 to 1 within one second.
- Counter-checks as unit tests: without the audio ramps, and without
  the level on the video sprite in the game, both go red.
- Gate on melektaus-atx: 309 files, 4796 tests passed, 8 more than
  at 77ac9d1. The first run failed 14 tests in 6 files that list the
  default sprite properties or the bus levels in full. They now
  expect videoVolume and the new level field of busLevels.
- Not tried in a browser: video fades, pattern and song items with a
  fade.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/editor/composition/CompositionTimeline.ts

@@ -28,7 +28,11 @@ import {
   compositionLength,
   compositionLoops,
   fittedLength,
+  itemEnd,
+  itemFades,
+  type ItemFades,
   itemObject,
+  itemSounds,
   itemStands,
   itemVideo,
 } from '../../core/model/composition';
@@ -38,6 +42,7 @@ import {
   clipSpans,
   droppedLength,
   droppedSpan,
+  fadedItem,
   itemMedia,
   type ItemMedia,
   heldMove,
@@ -55,7 +60,7 @@ import { placementRejection } from '../../core/model/placement';
 import type { Vec2 } from '../../core/model/Properties';
 import { findAsset, findAssetOf } from '../../core/model/Project';
 import { objectName } from '../../core/model/SceneObject';
-import { findSound } from '../../core/model/sound';
+import { fadeGain, findSound } from '../../core/model/sound';
 import { assetRefLabel } from '../../core/properties/assetRef';
 import { playedSoundById } from '../../core/properties/playedSound';
 import {
@@ -93,6 +98,7 @@ import {
   clipBox,
   clipsInBand,
   type CompositionView,
+  fadeHandles,
   HEAD_WIDTH,
   type Hit,
   hitAt,
@@ -123,6 +129,9 @@ const AUDIO_COLORS: Record<CompositionAudioKind, number> = {
 const DEFAULT_VIEW: CompositionView = { panX: 0, pxPerTick: 2, panY: 0, trackHeight: TRACK_HEIGHT };
 /** Room at the top of an item for its name; marks of its video keep below it. */
 const LABEL_STRIP = 15;
+/** The side of a fade handle, and the most line pieces a fade's curve is drawn with. */
+const FADE_HANDLE = 7;
+const FADE_STEPS = 32;
 const CLIP_LABEL = textStyle('small').clone();
 CLIP_LABEL.fill = Theme.colors.text;
 
@@ -143,7 +152,9 @@ type Gesture =
       /** Alt held while dragging items: they go without their clips. */
       alone: boolean;
     }
-  | { kind: 'resize' | 'trim'; anchor: CompositionItem; item: CompositionItem; media: ItemMedia | null };
+  | { kind: 'resize' | 'trim'; anchor: CompositionItem; item: CompositionItem; media: ItemMedia | null }
+  /** The handle of a fade dragged along the top of its item. */
+  | { kind: 'fade'; end: 'in' | 'out'; anchor: CompositionItem; item: CompositionItem };
 
 /** An asset dragged in from the panel: an item or, for an animation, a clip – and whether it fits there. */
 interface Drop {
@@ -198,8 +209,10 @@ export class CompositionTimeline extends Widget {
   private assetId = '';
   private gesture: Gesture | null = null;
   private drop: Drop | null = null;
-  /** The end of an item the pointer would drag; it lights up, and the cursor says so. */
+  /** The end or fade handle of an item the pointer would drag; it lights up, and the cursor says so. */
   private hover: { id: string; part: Part } | null = null;
+  /** The item under the pointer: its fade handles show. */
+  private pointed: string | null = null;
   /** Where the pointer was last, in local coordinates: a paste lands there while it is over the lanes. */
   private pointer: Vec2 | null = null;
   /** Sound files whose waveform is on its way. */
@@ -552,7 +565,13 @@ export class CompositionTimeline extends Widget {
   }
 
   private hitAt(asset: CompositionAsset, g: Vec2): Hit | null {
-    return hitAt(this.view, asset.tracks, this.liveItems(asset), asset.clips, this.lengthOf(asset), g);
+    const fades = (item: CompositionItem) => this.fadesOf(asset, item);
+    return hitAt(this.view, asset.tracks, this.liveItems(asset), asset.clips, this.lengthOf(asset), g, fades);
+  }
+
+  /** The fades of an item that has a sound for them to shape; null for one without. */
+  private fadesOf(asset: CompositionAsset, item: CompositionItem): ItemFades | null {
+    return itemSounds(this.state.project, asset, item) ? itemFades(item) : null;
   }
 
   private onPointerDown(e: FederatedPointerEvent): void {
@@ -626,6 +645,11 @@ export class CompositionTimeline extends Widget {
       return;
     }
     if (!picked.includes(item.id) || hit.part !== 'body') this.pickItems([item.id]);
+    if (hit.part === 'fadeIn' || hit.part === 'fadeOut') {
+      const end = hit.part === 'fadeIn' ? 'in' : 'out';
+      this.gesture = { kind: 'fade', end, anchor: item, item };
+      return;
+    }
     const media = itemMedia(this.state.project, asset, item);
     if (hit.part === 'end') this.gesture = { kind: 'resize', anchor: item, item, media };
     else if (hit.part === 'start') this.gesture = { kind: 'trim', anchor: item, item, media };
@@ -718,6 +742,9 @@ export class CompositionTimeline extends Widget {
         gesture.item = resizedItem(gesture.anchor, end - gesture.anchor.start);
         break;
       }
+      case 'fade':
+        gesture.item = fadedItem(gesture.anchor, gesture.end, frame);
+        break;
       case 'trim':
         gesture.item = trimmedItem(
           gesture.anchor,
@@ -828,11 +855,16 @@ export class CompositionTimeline extends Widget {
       }
       case 'resize':
       case 'trim':
+      case 'fade':
         if (JSON.stringify(gesture.item) !== JSON.stringify(gesture.anchor))
           this.commit(
             asset,
             { items: asset.items.map((i) => (i.id === gesture.anchor.id ? gesture.item : i)) },
-            gesture.kind === 'resize' ? 'Resize item' : 'Trim item',
+            gesture.kind === 'fade'
+              ? `Set fade ${gesture.end}`
+              : gesture.kind === 'resize'
+                ? 'Resize item'
+                : 'Trim item',
           );
         break;
       default:
@@ -944,15 +976,20 @@ export class CompositionTimeline extends Widget {
     this.redraw();
   }
 
-  /** The end of an item under the pointer it can be dragged by: a resize cursor, and the end lights up. */
+  /**
+   * The end or fade handle of an item under the pointer it can be dragged by: a resize cursor, and
+   * the end or handle lights up. The handles of the item under the pointer show.
+   */
   private followHover(asset: CompositionAsset, local: Vec2, g: Vec2): void {
     const inside = local.x >= HEAD_WIDTH && local.x <= this.w && local.y >= RULER_HEIGHT && local.y <= this.h;
     const hit = inside ? this.hitAt(asset, g) : null;
     const hover = hit?.kind === 'item' && hit.part !== 'body' ? { id: hit.id, part: hit.part } : null;
+    const pointed = hit?.kind === 'item' ? hit.id : null;
     const cursor = hover ? 'ew-resize' : 'default';
     if (this.cursor !== cursor) this.cursor = cursor;
-    if (JSON.stringify(hover) === JSON.stringify(this.hover)) return;
+    if (JSON.stringify(hover) === JSON.stringify(this.hover) && pointed === this.pointed) return;
     this.hover = hover;
+    this.pointed = pointed;
     this.redraw();
   }
 
@@ -1139,7 +1176,7 @@ export class CompositionTimeline extends Widget {
       );
       return moved ?? live;
     }
-    if (g?.kind === 'resize' || g?.kind === 'trim')
+    if (g?.kind === 'resize' || g?.kind === 'trim' || g?.kind === 'fade')
       return live.map((i) => (i.id === g.anchor.id ? g.item : i));
     return live;
   }
@@ -1268,6 +1305,8 @@ export class CompositionTimeline extends Widget {
         });
       if (item.audio?.kind === 'sound') this.drawWave(item, box);
       if (media) this.drawMediaMarks(item, media, box);
+      const fades = this.fadesOf(asset, item);
+      if (fades) this.drawFades(item, fades, box);
       const object = itemObject(asset, item);
       const placed = object && findAsset(this.state.project, object.assetId);
       const name = item.audio
@@ -1280,8 +1319,71 @@ export class CompositionTimeline extends Widget {
         labels.push({ text: name, x: Math.max(box.x, HEAD_WIDTH) + 4, y: box.y + 2, w: seen - 8 });
     }
     this.drawOverlaps(shown, rows);
+    const part = this.hover?.part;
     const hover = this.hover && boxes.get(this.hover.id);
-    if (hover && this.hover!.part !== 'body') this.drawGrip(hover, this.hover!.part);
+    if (hover && (part === 'start' || part === 'end')) this.drawGrip(hover, part);
+    this.drawFadeHandles(asset, shown, rows, picked);
+  }
+
+  /**
+   * The fades of an item as the level its sound rises and falls along, from its bottom edge to its top:
+   * what lies above is shaded, as the part not heard.
+   */
+  private drawFades(item: CompositionItem, fades: ItemFades, box: Box): void {
+    const ramp = (from: number, frames: number, rising: boolean, curve: ItemFades['curveIn']) => {
+      const x0 = HEAD_WIDTH + tickToX(this.view, from);
+      const x1 = HEAD_WIDTH + tickToX(this.view, from + frames);
+      const steps = Math.max(1, Math.min(FADE_STEPS, Math.ceil((x1 - x0) / 3)));
+      const points = Array.from({ length: steps + 1 }, (_, i) => {
+        const t = i / steps;
+        const level = fadeGain(curve, rising ? t : 1 - t);
+        return { x: x0 + (x1 - x0) * t, y: box.y + box.h * (1 - level) };
+      });
+      const corner = rising ? { x: x0, y: box.y } : { x: x1, y: box.y };
+      this.lanes.poly([corner, ...points].flatMap((p) => [p.x, p.y])).fill({ color: 0x000000, alpha: 0.35 });
+      this.lanes.moveTo(points[0]!.x, points[0]!.y);
+      for (const p of points.slice(1)) this.lanes.lineTo(p.x, p.y);
+      this.lanes.stroke({ width: 1, color: 0xffffff, alpha: 0.7 });
+    };
+    if (fades.fadeIn > 0) ramp(item.start, fades.fadeIn, true, fades.curveIn);
+    if (fades.fadeOut > 0) ramp(itemEnd(item) - fades.fadeOut, fades.fadeOut, false, fades.curveOut);
+  }
+
+  /**
+   * The handles of the fades on the top edge of an item: of the one under the pointer, the picked
+   * ones, and every one with a fade; the one the pointer would drag lit.
+   */
+  private drawFadeHandles(
+    asset: CompositionAsset,
+    shown: readonly CompositionItem[],
+    rows: ReadonlyMap<string, number>,
+    picked: ReadonlySet<string>,
+  ): void {
+    const g = this.gesture;
+    for (const item of shown) {
+      const row = rows.get(item.track);
+      const fades = row !== undefined ? this.fadesOf(asset, item) : null;
+      const handles = fades && fadeHandles(this.view, row!, item, fades);
+      if (!handles) continue;
+      const shows = item.id === this.pointed || picked.has(item.id) || g?.kind === 'fade';
+      const y = RULER_HEIGHT + handles.y;
+      for (const [end, x, frames] of [
+        ['in', handles.inX, fades.fadeIn],
+        ['out', handles.outX, fades.fadeOut],
+      ] as const) {
+        if (!shows && frames === 0) continue;
+        const lit =
+          (g?.kind === 'fade' && g.anchor.id === item.id && g.end === end) ||
+          (this.hover?.id === item.id && this.hover.part === (end === 'in' ? 'fadeIn' : 'fadeOut'));
+        const at = HEAD_WIDTH + x;
+        if (at < HEAD_WIDTH || at > this.w) continue;
+        const half = FADE_HANDLE / 2;
+        this.over
+          .rect(Math.round(at - half), y + 1, FADE_HANDLE, FADE_HANDLE)
+          .fill(lit ? Theme.colors.accentText : 0xffffff)
+          .stroke({ width: 1, color: 0x000000, alpha: 0.6 });
+      }
+    }
   }
 
   /**

src/editor/composition/compositionView.ts

@@ -43,19 +43,42 @@ export function clipBox(view: CompositionView, row: number, span: { start: numbe
   };
 }
 
-export type Part = 'start' | 'body' | 'end';
+export type Part = 'start' | 'body' | 'end' | 'fadeIn' | 'fadeOut';
 
 /** What lies under a point of the lanes: a clip before an item, as clips are drawn over items. */
 export type Hit = { kind: 'clip'; id: string } | { kind: 'item'; id: string; part: Part };
 
+/** How near a fade handle the pointer takes it, across and down from the top of its item. */
+const FADE_REACH = 6;
+
+/**
+ * Where the handles of an item's fades sit, in lane coordinates: on its top edge, at the end of the
+ * fade in and the start of the fade out. None on an item too narrow to hold both apart from its ends.
+ */
+export function fadeHandles(
+  view: CompositionView,
+  row: number,
+  item: CompositionItem,
+  fades: { fadeIn: number; fadeOut: number },
+): { inX: number; outX: number; y: number } | null {
+  const box = itemBox(view, row, item);
+  if (box.w < FADE_REACH * 6 || box.h < FADE_REACH * 2) return null;
+  return {
+    inX: tickToX(view, item.start + fades.fadeIn),
+    outX: tickToX(view, item.start + item.length - fades.fadeOut),
+    y: box.y,
+  };
+}
+
 /** Items in the order they are drawn: a later start over an earlier one, as in the picture. */
 export function stacked<T extends { start: number }>(items: readonly T[]): T[] {
   return [...items].sort((a, b) => a.start - b.start);
 }
 
 /**
- * What lies under a point. Of items an edge within reach comes first, the nearest one – also the end
- * of one an overlapping item covers, so it can still be dragged –, else the body drawn on top.
+ * What lies under a point. Of items a fade handle comes first – `fades` says which items have them –,
+ * then an edge within reach, the nearest one – also the end of one an overlapping item covers, so it
+ * can still be dragged –, else the body drawn on top.
  */
 export function hitAt(
   view: CompositionView,
@@ -64,6 +87,7 @@ export function hitAt(
   clips: readonly CompositionClip[],
   clipLength: (clip: CompositionClip) => number,
   p: Vec2,
+  fades: (item: CompositionItem) => { fadeIn: number; fadeOut: number } | null = () => null,
 ): Hit | null {
   const rows = new Map(tracks.map((t, i) => [t.id, i]));
   const inside = (b: Box) => p.x >= b.x && p.x <= b.x + b.w && p.y >= b.y && p.y <= b.y + b.h;
@@ -74,6 +98,18 @@ export function hitAt(
       return { kind: 'clip', id: clip.id };
   }
   const order = stacked(items);
+  const top = [...order].reverse().find((item) => {
+    const row = rows.get(item.track);
+    return row !== undefined && inside(itemBox(view, row, item));
+  });
+  const faded = top && fades(top);
+  const handles = faded && fadeHandles(view, rows.get(top.track)!, top, faded);
+  if (handles && p.y <= handles.y + FADE_REACH * 2) {
+    const toIn = Math.abs(p.x - handles.inX);
+    const toOut = Math.abs(p.x - handles.outX);
+    if (Math.min(toIn, toOut) <= FADE_REACH)
+      return { kind: 'item', id: top.id, part: toIn <= toOut ? 'fadeIn' : 'fadeOut' };
+  }
   let edge: { hit: Hit; distance: number } | null = null;
   let body: Hit | null = null;
   for (let i = order.length - 1; i >= 0; i--) {

src/editor/inspector/compositionSection.ts

@@ -12,11 +12,14 @@ import {
   compositionFps,
   fittedLength,
   isCompositionSprite,
+  itemEnd,
+  itemFades,
   itemVideo,
 } from '../../core/model/composition';
-import { heldMove, movedWithClips, resizedItem } from '../../core/model/compositionEdit';
+import { fadedItem, heldMove, movedWithClips, resizedItem } from '../../core/model/compositionEdit';
 import { findAsset, findAssetOf, type Project } from '../../core/model/Project';
-import { findSound } from '../../core/model/sound';
+import { FADE_CURVES, findSound, isFadeCurve } from '../../core/model/sound';
+import { DEFAULT_FADE_CURVE } from '../../core/properties/sounds';
 import { ITEM_MIN_DB, SOUND_ITEM_MAX_DB } from '../../core/music/song';
 import type { PropertyDef } from '../../core/properties/PropertySchema';
 import { Button, formatNumber, type MenuEntry } from '../../ui';
@@ -32,6 +35,10 @@ const ANIMATION_ROW = 'composition:animation';
 const PLAYS_ROW = 'composition:plays';
 const VOLUME_ROW = 'composition:volume';
 const BUS_ROW = 'composition:bus';
+const FADE_ROWS = {
+  in: { key: 'composition:fadeIn', curve: 'composition:curveIn', label: 'Fade in' },
+  out: { key: 'composition:fadeOut', curve: 'composition:curveOut', label: 'Fade out' },
+} as const;
 
 const AUDIO_TITLES: Record<CompositionAudioKind, string> = {
   sound: 'SOUND ITEM',
@@ -262,6 +269,57 @@ export function addTimingRows(host: InspectorHost, holderId: string, objectId: s
   const fit = new Button('Fit to media', { onClick: () => fitToMedia(host, holderId, objectId) });
   fit.setEnabled(!(item.fitMedia === true && item.length === fittedLength(host.state.project, comp, item)));
   host.body.add(fit);
+  if (video) addFadeRows(host, item, fps, write);
+}
+
+/**
+ * How long its sound rises from silence at its start and falls back at its end, and along which
+ * curve; the handles on the item's top edge in the timeline set the same.
+ */
+function addFadeRows(
+  host: InspectorHost,
+  item: CompositionItem,
+  fps: number,
+  write: (label: string, edit: (current: CompositionItem) => CompositionItem) => void,
+): void {
+  const fades = itemFades(item);
+  for (const end of ['in', 'out'] as const) {
+    const { key, curve, label } = FADE_ROWS[end];
+    const frames = end === 'in' ? fades.fadeIn : fades.fadeOut;
+    const length = createFieldRow(host, seconds(key, label), frames / fps, {
+      onInput: () => undefined,
+      onChange: (value) =>
+        write(`Set fade ${end}`, (i) => {
+          const by = Math.round(Number(value) * fps) || 0;
+          return fadedItem(i, end, end === 'in' ? i.start + by : itemEnd(i) - by);
+        }),
+    });
+    const shape = createFieldRow(
+      host,
+      {
+        key: curve,
+        label: `${label} curve`,
+        kind: 'enum',
+        default: DEFAULT_FADE_CURVE,
+        options: FADE_CURVES,
+      },
+      end === 'in' ? fades.curveIn : fades.curveOut,
+      {
+        onInput: () => undefined,
+        onChange: (value) => {
+          if (!isFadeCurve(value)) return;
+          write(`Set fade ${end} curve`, (i) => ({ ...i, [end === 'in' ? 'curveIn' : 'curveOut']: value }));
+        },
+      },
+    );
+    for (const [rowKey, row] of [
+      [key, length],
+      [curve, shape],
+    ] as const) {
+      host.fields.set(rowKey, row);
+      host.body.add(row.row);
+    }
+  }
 }
 
 export function refreshTimingRows(host: InspectorHost, holderId: string, objectId: string): void {
@@ -271,6 +329,11 @@ export function refreshTimingRows(host: InspectorHost, holderId: string, objectI
   host.fields.get(START_ROW)?.setValue(found.item.start / fps);
   host.fields.get(LENGTH_ROW)?.setValue(found.item.length / fps);
   host.fields.get(OFFSET_ROW)?.setValue(found.item.offset);
+  const fades = itemFades(found.item);
+  host.fields.get(FADE_ROWS.in.key)?.setValue(fades.fadeIn / fps);
+  host.fields.get(FADE_ROWS.out.key)?.setValue(fades.fadeOut / fps);
+  host.fields.get(FADE_ROWS.in.curve)?.setValue(fades.curveIn);
+  host.fields.get(FADE_ROWS.out.curve)?.setValue(fades.curveOut);
 }
 
 /** What shapes the timing rows: whether there is a video, and whether the length still follows it. */