A game that stays silent is only half a game, so a sound is an asset like any other (#45)

Sounds follow the road fonts already paved: the file goes to the server
content-addressed under its sha256, the project keeps a named asset that
points at it, and nothing is ever re-encoded. The format is read from the
file's own signature, not from what the browser claims - the same .m4a
arrives as audio/mp4, audio/x-m4a or as nothing at all.

Playing is two nodes. "Play sound" takes the asset and may raise or lower
its volume; every call is its own voice, so two jumps in a row are two
jumps heard. Its loop input is a choice of three - from the asset, on,
off - because a checkbox has to spend one of its two positions on "not
set" and would lose a case: an asset that loops could then never be
played once.

Below the graph core stays browser-free: NodeContext hands out an asset
id, BehaviorSystem resolves it against the project and folds the asset's
own volume and loop into the call, and only viewport/SoundLibrary knows
about Web Audio. A run without a sink plays nothing instead of throwing.

The CI browser is muted, so the ear is no witness: the debug hook counts
started and stopped voices, and step 34 reads that counter.

tools/e2e/21-list.mjs

@@ -89,7 +89,9 @@ export async function listTab(t) {
   check(
     'a list of its own offers every kind, the struct included',
     (await json('__gm.list.info()')).offers.join(',') ===
-      'number,text,boolean,color,texture,statemachine,struct',
+      // In the order of the kind table (`core/properties/kinds.ts`) – a new kind is a row there,
+      // and `sound` (#45) is the newest of them.
+      'number,text,boolean,color,texture,statemachine,sound,struct',
     JSON.stringify((await json('__gm.list.info()')).offers),
   );
   await pickKind('Text');

tools/e2e/34-sound.mjs

@@ -0,0 +1,275 @@
+import { edge, node } from '../lib/builders.mjs';
+import { LANES, MAKE_OBJECT, pickAssetType } from './lib/scaffold.mjs';
+
+/**
+ * Sound assets (#45): uploaded like a typeface – picked from what the server holds through
+ * "+ → Sound", or dropped onto the editor – and played by a graph through `Play sound`. Nothing
+ * wears a sound and nothing places it; the asset says how loud it is and whether it loops, and the
+ * node scales the one and can overrule the other in either direction.
+ *
+ * Nothing is ever *heard* here: CI runs the browser muted (see tools/lib/browser.mjs). The witness
+ * is `__gm.sound.stats()` – voices started, stopped and in flight – the way `renderStats` is the
+ * witness for a repaint that did not happen. No pixel check could see any of this.
+ *
+ * The seed puts two generated WAVs on the server without making assets of them: a 0.25 s blip and
+ * a 0.75 s drone, told apart by their exact size rather than by their name.
+ */
+const LANE = LANES.sound;
+const BLIP_BYTES = 4044;
+const DRONE_BYTES = 12044;
+
+export async function sound(t) {
+  const { check, sleep, json, evaluate, mutate, click, clickText, press, play, stop, shot } = t;
+  const { select } = t;
+  const assetsOfType = (type) =>
+    json(`__gm.state.project.assets.filter((a) => a.type === ${JSON.stringify(type)})
+      .map((a) => ({ id: a.id, name: a.name, file: a.file, props: a.props }))`);
+  const stats = () => json('__gm.sound.stats()');
+  /** A voice starts only once its bytes have been decoded, so what it changes is awaited. */
+  const waitFor = async (read, done, tries = 25) => {
+    let value = await read();
+    for (let i = 0; i < tries && !done(value); i++) {
+      await sleep(150);
+      value = await read();
+    }
+    return value;
+  };
+
+  await press('Escape');
+  await evaluate('document.activeElement instanceof HTMLElement && document.activeElement.blur()');
+  await evaluate('__gm.state.select(null)');
+  await evaluate("__gm.showTab('scene')");
+  await sleep(200);
+
+  // 34a. What the server holds, straight from the API: the two files the seed put there.
+  const stored = await evaluate(`fetch('/game-maker/api/index.php?r=sounds').then((r) => r.json())`);
+  const blip = stored.find((item) => item.bytes === BLIP_BYTES);
+  const drone = stored.find((item) => item.bytes === DRONE_BYTES);
+  check(
+    'the server holds the two seeded sounds, named by their content',
+    Boolean(blip && drone) && /^[a-f0-9]{64}\.wav$/.test(blip.file),
+    JSON.stringify({ blip: blip?.file, drone: drone?.file, count: stored.length }),
+  );
+  if (!blip || !drone) return;
+  check(
+    'the listing carries the name each file was uploaded under',
+    blip.name === 'blip.wav' && drone.name === 'drone.wav',
+    JSON.stringify({ blip: blip.name, drone: drone.name }),
+  );
+
+  // 34b. "+ → Sound" offers them; a sound is picked rather than created empty, like a typeface.
+  await clickText('+');
+  await sleep(200);
+  await pickAssetType(t, 'Sound');
+  await sleep(500);
+  const picker = await json('__gm.soundPicker()');
+  check(
+    'the + menu opens the sound picker on what the server holds, none of it in the project yet',
+    Boolean(picker) && picker.files.includes(blip.file) && picker.used.length === 0,
+    JSON.stringify(picker),
+  );
+  check(
+    'a row nobody uses yet is captioned with the name the file was uploaded under',
+    picker.captions[picker.files.indexOf(blip.file)] === 'blip.wav',
+    JSON.stringify(picker.captions),
+  );
+  await shot('34-sound-picker');
+
+  // 34c. The ▶ at a row's left is a preview: it plays that file without choosing it.
+  const playAt = await json(`__gm.soundPickerPlay(${JSON.stringify(drone.file)})`);
+  await click(playAt.x, playAt.y);
+  const previewed = await waitFor(stats, (s) => s.started > 0);
+  const afterPreview = await json('__gm.soundPicker()');
+  check(
+    'pressing a row’s ▶ plays that file and picks nothing',
+    previewed.started === 1 &&
+      previewed.last?.file === drone.file &&
+      afterPreview.played === drone.file &&
+      afterPreview.selected === null,
+    JSON.stringify({ previewed, selected: afterPreview.selected, played: afterPreview.played }),
+  );
+
+  const row = await json(`__gm.soundPickerRow(${JSON.stringify(blip.file)})`);
+  await click(row.x, row.y);
+  await sleep(200);
+  await clickText('Use sound');
+  await sleep(400);
+  const sounds = await assetsOfType('sound');
+  check(
+    'picking a row makes a sound asset of that file, named after the file rather than "Sound"',
+    sounds.length === 1 && sounds[0].file === blip.file && sounds[0].name === 'blip',
+    JSON.stringify(sounds),
+  );
+  if (sounds.length !== 1) return;
+  const soundId = sounds[0].id;
+  check(
+    'a fresh sound asset is full volume and one-shot',
+    sounds[0].props.volume === 1 && sounds[0].props.loop === false,
+    JSON.stringify(sounds[0].props),
+  );
+  // Closing the picker silences its preview; nothing it started may outlive the dialog.
+  check(
+    'closing the picker leaves nothing playing',
+    (await stats()).playing === 0,
+    JSON.stringify(await stats()),
+  );
+
+  // 34d. Its inspector: the two rows, and the preview that is the only way to tell which sound it is.
+  await select({ kind: 'asset', assetId: soundId });
+  await sleep(300);
+  const volumeRow = await json("__gm.inspectorField('Volume', true)");
+  const loopRow = await json("__gm.inspectorField('Loop', true)");
+  check(
+    'a sound asset has a Volume row and a Loop row',
+    Boolean(volumeRow) && Boolean(loopRow),
+    JSON.stringify({ volumeRow, loopRow }),
+  );
+  await shot('34-sound-asset');
+
+  // 34e. A graph plays it. The behavior sits in its own lane; the node's volume is a factor on the
+  //      asset's, so 0.5 × 0.4 has to reach the library as 0.2 rather than as either of the two.
+  await mutate(`
+    const p = __gm.state.project;
+    const scene = __gm.state.activeScene;
+    p.assets.find((a) => a.id === ${JSON.stringify(soundId)}).props.volume = 0.4;
+    const asset = structuredClone(p.assets.find((a) => a.id === 'mover'));
+    Object.assign(asset, {
+      id: 'sn-logic',
+      name: 'Noisy',
+      props: { enabled: true, runInEditor: false },
+      variables: [],
+      graph: ${JSON.stringify({
+        nodes: [
+          node('sn-start', 'OnStart', 0, 0),
+          node('sn-play', 'PlaySound', 260, 0, { sound: soundId, volume: 0.5, loop: '' }),
+        ],
+        edges: [edge('sn-e1', 'sn-start', 'exec', 'sn-play', 'exec')],
+      })},
+    });
+    p.assets.unshift(asset);
+    ${MAKE_OBJECT}
+    scene.objects.push(
+      make('sn-box', 'crate', '', ${LANE}, 'Noisy box'),
+      make('sn-brain', 'sn-logic', 'sn-box', 0, 'Noisy'),
+    );
+    __gm.lookAt(0, ${LANE});`);
+  await sleep(400);
+
+  await evaluate('__gm.sound.stopAll()');
+  const before = await stats();
+  await play();
+  const played = await waitFor(stats, (s) => s.started > before.started);
+  check(
+    'a graph playing a sound reaches the library, at the asset volume scaled by the node’s',
+    played.started === before.started + 1 &&
+      played.last?.file === blip.file &&
+      Math.abs(played.last.volume - 0.2) < 0.001 &&
+      played.last.loop === false,
+    JSON.stringify(played),
+  );
+  await stop();
+  await sleep(300);
+
+  // 34f. A looping sound is music: it plays on, and ending the run ends it – nothing may outlive
+  //      the world that started it.
+  await mutate(`
+    const p = __gm.state.project;
+    p.assets.find((a) => a.id === ${JSON.stringify(soundId)}).props.loop = true;`);
+  await sleep(300);
+  await play();
+  const looping = await waitFor(stats, (s) => s.playing > 0 && s.last?.loop === true);
+  check(
+    'the asset’s own loop flag reaches the library even though the node never asked for one',
+    looping.playing >= 1 && looping.last?.loop === true,
+    JSON.stringify(looping),
+  );
+  // Well past the blip's own 0.25 s, so a one-shot would have ended by itself here.
+  await sleep(700);
+  const stillLooping = await stats();
+  await stop();
+  await sleep(400);
+  const afterStop = await stats();
+  check(
+    'a loop keeps playing while the run is on and ends with it',
+    stillLooping.playing >= 1 && afterStop.playing === 0,
+    JSON.stringify({ stillLooping, afterStop }),
+  );
+
+  // 34f-2. The node overrules the asset in both directions – the reason `loop` is a choice of three
+  //        and not a checkbox: music has to be playable as a one-shot.
+  const setNodeLoop = (value) =>
+    mutate(`
+      const p = __gm.state.project;
+      const graph = p.assets.find((a) => a.id === 'sn-logic').graph;
+      graph.nodes.find((n) => n.id === 'sn-play').params.loop = ${JSON.stringify(value)};`);
+  await setNodeLoop('false');
+  await sleep(300);
+  const beforeOnce = await stats();
+  await play();
+  const once = await waitFor(stats, (s) => s.started > beforeOnce.started);
+  check('a node saying "off" plays a looping asset once', once.last?.loop === false, JSON.stringify(once));
+  await stop();
+  await sleep(300);
+  await setNodeLoop('');
+  await sleep(300);
+
+  // 34g. `Stop sound` silences it from inside the graph, without ending the run.
+  await mutate(`
+    const p = __gm.state.project;
+    const graph = p.assets.find((a) => a.id === 'sn-logic').graph;
+    graph.nodes.push(${JSON.stringify(node('sn-stop', 'StopSound', 520, 0, { sound: soundId }))});
+    graph.nodes.push(${JSON.stringify(node('sn-key', 'OnKeyDown', 260, 220, { key: 's' }))});
+    graph.edges.push(${JSON.stringify(edge('sn-e2', 'sn-key', 'exec', 'sn-stop', 'exec'))});`);
+  await sleep(300);
+  await play();
+  await waitFor(stats, (s) => s.playing > 0);
+  await press('s');
+  const silenced = await waitFor(stats, (s) => s.playing === 0);
+  check(
+    'Stop sound silences a looping sound while the run carries on',
+    silenced.playing === 0 && (await json('__gm.playInfo()')).playing === true,
+    JSON.stringify(silenced),
+  );
+  await stop();
+  await sleep(300);
+
+  // 34h. Dropping a sound file onto the editor uploads it and makes an asset named after the file.
+  const dropped = await evaluate(`
+    (async () => {
+      const bytes = await fetch('/game-maker/data/sounds/${drone.file}').then((r) => r.arrayBuffer());
+      const file = new File([bytes], 'drone.wav', { type: 'audio/wav' });
+      const data = new DataTransfer();
+      data.items.add(file);
+      const canvas = document.querySelector('canvas');
+      canvas.dispatchEvent(new DragEvent('drop', { dataTransfer: data, bubbles: true, cancelable: true }));
+      return file.size;
+    })()`);
+  await waitFor(
+    () => assetsOfType('sound'),
+    (list) => list.length === 2,
+  );
+  const both = await assetsOfType('sound');
+  check(
+    'a dropped sound is uploaded and becomes an asset named after the file',
+    dropped === DRONE_BYTES &&
+      both.length === 2 &&
+      both.some((a) => a.file === drone.file && a.name === 'drone'),
+    JSON.stringify({ dropped, both, notice: await json('__gm.state.notice') }),
+  );
+
+  // 34i. The step takes back everything it made, so the steps after it see the fixture again.
+  await mutate(`
+    const gone = new Set(['sn-logic', ...${JSON.stringify(both.map((a) => a.id))}]);
+    const project = __gm.state.project;
+    project.assets = project.assets.filter((a) => !gone.has(a.id));
+    __gm.state.activeScene.objects = __gm.state.activeScene.objects.filter(
+      (o) => !['sn-box', 'sn-brain'].includes(o.id));
+    __gm.state.select(null);
+    __gm.lookAt(0, 0);`);
+  await sleep(300);
+  check(
+    'the step leaves nothing of its own behind',
+    (await assetsOfType('sound')).length === 0 && (await stats()).playing === 0,
+    JSON.stringify(await assetsOfType('sound')),
+  );
+}

tools/e2e/index.mjs

@@ -36,6 +36,7 @@ import { font } from './30-font.mjs';
 import { particles } from './31-particles.mjs';
 import { scope } from './32-scope.mjs';
 import { renderOnChange } from './33-render-on-change.mjs';
+import { sound } from './34-sound.mjs';
 
 export const STEPS = [
   hierarchy,
@@ -71,4 +72,5 @@ export const STEPS = [
   particles,
   scope,
   renderOnChange,
+  sound,
 ];

tools/e2e/lib/scaffold.mjs

@@ -19,6 +19,7 @@ export const LANES = {
   transform: -1500,
   font: -1900,
   particles: -2300,
+  sound: -2700,
 };
 
 /**