A game that stays silent is only half a game, so a sound is an asset like any other (#45)

Sounds follow the road fonts already paved: the file goes to the server
content-addressed under its sha256, the project keeps a named asset that
points at it, and nothing is ever re-encoded. The format is read from the
file's own signature, not from what the browser claims - the same .m4a
arrives as audio/mp4, audio/x-m4a or as nothing at all.

Playing is two nodes. "Play sound" takes the asset and may raise or lower
its volume; every call is its own voice, so two jumps in a row are two
jumps heard. Its loop input is a choice of three - from the asset, on,
off - because a checkbox has to spend one of its two positions on "not
set" and would lose a case: an asset that loops could then never be
played once.

Below the graph core stays browser-free: NodeContext hands out an asset
id, BehaviorSystem resolves it against the project and folds the asset's
own volume and loop into the call, and only viewport/SoundLibrary knows
about Web Audio. A run without a sink plays nothing instead of throwing.

The CI browser is muted, so the ear is no witness: the debug hook counts
started and stopped voices, and step 34 reads that counter.

src/core/commands/assetCommands.ts

@@ -1,8 +1,9 @@
 import { createEmptyGraph } from '../graph/Graph';
 import { newId } from '../ids';
-import type { Asset, FontAsset, StateMachineAsset, TextureAsset } from '../model/Asset';
+import type { Asset, FontAsset, SoundAsset, StateMachineAsset, TextureAsset } from '../model/Asset';
 import type { AssetType } from '../model/AssetTypes';
 import { isFontFile } from '../model/font';
+import { isSoundFile } from '../model/sound';
 import { removeSubtrees } from '../model/hierarchy';
 import type { PropertyBag, PropertyValue } from '../model/Properties';
 import { objectHolders, type Project } from '../model/Project';
@@ -44,6 +45,8 @@ export function buildAsset(type: AssetType, name: string, id = newId()): Asset {
       return { ...base, type: 'shader', graph: createEmptyGraph() };
     case 'font':
       return { ...base, type: 'font', file: '' };
+    case 'sound':
+      return { ...base, type: 'sound', file: '' };
     case 'particles':
       return { ...base, type: 'particles', graph: createEmptyGraph() };
   }
@@ -156,6 +159,31 @@ export class CreateFontCommand implements Command {
   }
 }
 
+export function buildSoundAsset(name: string, file: string, id = newId()): SoundAsset {
+  return { id, name, props: defaultsFor('sound'), type: 'sound', file };
+}
+
+/** The sound is uploaded before the command runs, as a typeface is for `CreateFontCommand`. */
+export class CreateSoundCommand implements Command {
+  readonly label: string;
+  readonly asset: SoundAsset;
+
+  constructor(name: string, file: string) {
+    this.asset = buildSoundAsset(name, file);
+    this.label = `Import sound "${name}"`;
+  }
+
+  execute(project: Project): void {
+    if (!isSoundFile(this.asset.file))
+      throw new CommandError(`"${this.asset.file}" is not a stored sound file`);
+    project.assets.push(this.asset);
+  }
+
+  undo(project: Project): void {
+    project.assets = removeById(project.assets, this.asset.id);
+  }
+}
+
 /**
  * Deletes an asset and, as a cascade, every scene object referencing it, together with the objects
  * parented below those – in container templates as well as in scenes.

src/core/graph/NodeContext.ts

@@ -119,6 +119,16 @@ export interface NodeContext {
    */
   moveAndSlide(target: EntityHandle, dx: number, dy: number): { blockedX: boolean; blockedY: boolean };
   log(message: string): void;
+  /**
+   * Starts a sound asset, named by its id. `volume` multiplies the asset's own (1 leaves it alone);
+   * `loop` is the caller's answer to whether it repeats, and `undefined` leaves that to the asset,
+   * because the asset says what a sound *is* and the node only colours it. Every call is a voice of
+   * its own: two plays overlap, whether they loop or not, which is what makes one sound effect
+   * fired twice in a row audible twice.
+   */
+  playSound(assetId: string, volume: number, loop: boolean | undefined): void;
+  /** Silences every voice of that sound; `''` silences everything this run started. */
+  stopSound(assetId: string): void;
   /**
    * The position is local to `parent` (world when none); null when the asset does not exist or
    * cannot be placed. A spawned behavior runs its `On start` on the next tick.

src/core/graph/nodes/action/PlaySound.ts

@@ -0,0 +1,54 @@
+import type { NodeDef, PortDef } from '../../NodeRegistry';
+import { asNumber, asString, type Value } from '../../values';
+import { exec, num, soundPort, withDefault } from '../shared';
+
+/**
+ * Three answers, not two: a checkbox cannot say "leave it to the asset", and without that a sound
+ * whose own `Loop` is on could never be played as a one-shot.
+ */
+const LOOP_OPTIONS = [
+  { value: '', label: 'From the asset' },
+  { value: 'true', label: 'On' },
+  { value: 'false', label: 'Off' },
+];
+
+/** `any`, not `boolean`, so a graph can still drive the choice – see `loopChoice`. */
+const loopPort = (): PortDef => ({
+  key: 'loop',
+  label: 'loop',
+  type: 'any',
+  default: '',
+  field: { key: 'loop', label: 'Loop', kind: 'enum', default: '', options: LOOP_OPTIONS },
+});
+
+/**
+ * What the node says about looping: nothing (the asset decides) or an answer that overrules it.
+ * A boolean arriving over an edge is taken literally, which is what wiring a comparison or a
+ * variable into this input is for.
+ */
+export function loopChoice(value: Value | undefined): boolean | undefined {
+  if (typeof value === 'boolean') return value;
+  const text = asString(value);
+  if (text === 'true') return true;
+  if (text === 'false') return false;
+  return undefined;
+}
+
+export const PlaySound: NodeDef = {
+  type: 'PlaySound',
+  label: 'Play sound',
+  category: 'action',
+  description:
+    'Plays a sound asset. "volume" multiplies the volume the asset itself carries and "loop" either ' +
+    "leaves the asset's own setting alone or overrules it, so the asset says what the sound is and " +
+    'the node only colours it. Each play is a voice of its own: firing this twice in a row is heard ' +
+    'twice, overlapping.',
+  inputs: [exec(), soundPort(), withDefault(num('volume'), 1), loopPort()],
+  outputs: [exec()],
+  params: [],
+  run: (ctx, inputs) => {
+    const sound = asString(inputs.sound);
+    if (sound) ctx.playSound(sound, asNumber(inputs.volume), loopChoice(inputs.loop));
+    return { next: 'exec' };
+  },
+};

src/core/graph/nodes/action/StopSound.ts

@@ -0,0 +1,19 @@
+import type { NodeDef } from '../../NodeRegistry';
+import { asString } from '../../values';
+import { exec, soundPort } from '../shared';
+
+export const StopSound: NodeDef = {
+  type: 'StopSound',
+  label: 'Stop sound',
+  category: 'action',
+  description:
+    'Silences every voice of a sound, however many plays started it – including a looping one. ' +
+    'With no sound picked it silences everything the run is playing.',
+  inputs: [exec(), soundPort()],
+  outputs: [exec()],
+  params: [],
+  run: (ctx, inputs) => {
+    ctx.stopSound(asString(inputs.sound));
+    return { next: 'exec' };
+  },
+};

src/core/graph/nodes/data/literals.ts

@@ -35,6 +35,11 @@ export const StateMachineLiteral = literal(
   'statemachine',
   'A constant state machine, for a "State branch" or a "Set state" to work on.',
 );
+export const SoundLiteral = literal(
+  'Sound',
+  'sound',
+  'A constant sound, for a "Play sound" or a "Stop sound" to work on.',
+);
 export const ListLiteral = literal(
   'List',
   'list',

src/core/graph/nodes/index.ts

@@ -3,10 +3,12 @@ import { Destroy } from './action/Destroy';
 import { Emit } from './action/Emit';
 import { Log } from './action/Log';
 import { Move } from './action/Move';
+import { PlaySound } from './action/PlaySound';
 import { RestartGame } from './action/RestartGame';
 import { SetPosition } from './action/SetPosition';
 import { Spawn } from './action/Spawn';
 import { StopGame } from './action/StopGame';
+import { StopSound } from './action/StopSound';
 import { Abs } from './data/Abs';
 import { Add } from './data/Add';
 import { And } from './data/And';
@@ -32,6 +34,7 @@ import {
   BooleanLiteral,
   ListLiteral,
   NumberLiteral,
+  SoundLiteral,
   StateMachineLiteral,
   StringLiteral,
   TextureLiteral,
@@ -146,6 +149,7 @@ export const BUILT_IN_NODES: readonly NodeDef[] = [
   StringLiteral,
   TextureLiteral,
   StateMachineLiteral,
+  SoundLiteral,
   Add,
   Subtract,
   Multiply,
@@ -195,6 +199,8 @@ export const BUILT_IN_NODES: readonly NodeDef[] = [
   Spawn,
   Destroy,
   Emit,
+  PlaySound,
+  StopSound,
   Log,
   StopGame,
   RestartGame,

src/core/graph/nodes/shared.ts

@@ -36,6 +36,17 @@ export const keyParam: PropertyDef = { key: 'key', label: 'Key', kind: 'key', de
 /** No `assetTypes`: any placeable asset is a valid answer here. */
 export const assetParam: PropertyDef = { key: 'asset', label: 'Asset', kind: 'asset', default: '' };
 
+/**
+ * The `sound` input of the two sound nodes; unconnected, it shows the sound picker – the way
+ * `machinePort` shows the machine picker. A `Sound` literal is what feeds it through an edge.
+ */
+export const soundPort = (): PortDef => ({
+  key: 'sound',
+  label: 'sound',
+  type: 'string',
+  field: { key: 'sound', label: 'Sound', kind: 'sound', default: '' },
+});
+
 export function binaryMath(type: string, description: string, fn: (a: number, b: number) => number): NodeDef {
   return {
     type,

src/core/model/Asset.ts

@@ -129,6 +129,16 @@ export interface FontAsset extends AssetBase {
   file: string;
 }
 
+/**
+ * `file` names the sound in `data/sounds/` the way a font names its typeface: content-addressed
+ * `{sha256}.{ext}`, empty when the upload failed. Nothing wears a sound – a graph plays it with
+ * `Play sound`, and `props` says how loud and whether it repeats (see `properties/sounds.ts`).
+ */
+export interface SoundAsset extends AssetBase {
+  type: 'sound';
+  file: string;
+}
+
 /**
  * The graph is the motion of one particle: every output of its `Output` node is a function of the
  * particle's age and its own random number (see `core/particles/compile.ts`). A placed object is
@@ -153,6 +163,7 @@ export type Asset =
   | StateMachineAsset
   | ShaderAsset
   | FontAsset
+  | SoundAsset
   | ParticlesAsset;
 
 /** The assets edited as a node graph – what the graph canvas and the graph commands work on. */

src/core/model/AssetTypes.ts

@@ -12,6 +12,7 @@ export const ASSET_TYPES = [
   'statemachine',
   'shader',
   'font',
+  'sound',
   'particles',
 ] as const;
 
@@ -31,17 +32,19 @@ export const ASSET_TYPE_LABELS: Record<AssetType, string> = {
   statemachine: 'State machine',
   shader: 'Shader',
   font: 'Font',
+  sound: 'Sound',
   particles: 'Particles',
 };
 
 /**
  * Textures and animations are not placeable: they are pictures a sprite wears through `texture`,
  * as a shader is worn through `shader` and a font through `font`. A state machine is not either:
- * a behavior refers to it by id, and a Variables asset is declarations a graph reads by id.
+ * a behavior refers to it by id, and a Variables asset is declarations a graph reads by id. A
+ * sound is nowhere on screen at all – a graph plays it, so there is nothing to place.
  */
 export type PlaceableAssetType = Exclude<
   AssetType,
-  'scene' | 'texture' | 'animation' | 'statemachine' | 'shader' | 'font' | 'variables'
+  'scene' | 'texture' | 'animation' | 'statemachine' | 'shader' | 'font' | 'sound' | 'variables'
 >;
 
 const NOT_PLACEABLE: ReadonlySet<AssetType> = new Set([
@@ -51,6 +54,7 @@ const NOT_PLACEABLE: ReadonlySet<AssetType> = new Set([
   'statemachine',
   'shader',
   'font',
+  'sound',
   'variables',
 ]);
 

src/core/model/sound.ts

@@ -0,0 +1,20 @@
+import type { SoundAsset } from './Asset';
+import { findAsset, type Project } from './Project';
+
+/** Mirrors `SoundStore::isValidName` on the server, where the name is derived from the bytes. */
+const FILE_PATTERN = /^[a-f0-9]{64}\.(mp3|ogg|wav|m4a)$/;
+
+export function isSoundFile(name: string): boolean {
+  return FILE_PATTERN.test(name);
+}
+
+/** The file a `sound` value points at, empty when the id is unset or no longer resolves. */
+export function soundFileOf(project: Project, id: string): string {
+  if (id === '') return '';
+  const asset = findAsset(project, id);
+  return asset?.type === 'sound' ? asset.file : '';
+}
+
+export function isSoundAsset(asset: { type: string }): asset is SoundAsset {
+  return asset.type === 'sound';
+}

src/core/properties/PropertySchema.ts

@@ -40,6 +40,7 @@ import {
   TIME_SCALE_KEY,
 } from './particles';
 import { SHADER_KEY, shaderOf, shaderParamDefs } from './shaders';
+import { LOOP_KEY as SOUND_LOOP_KEY, VOLUME_KEY as SOUND_VOLUME_KEY } from './sounds';
 
 /**
  * What a kind's definition carries beyond key, label and default. `decimals` belongs only on
@@ -222,6 +223,12 @@ const ASSET_SCHEMAS: Record<AssetType, PropertySchema> = {
   shader: [],
   // The typeface itself is intrinsic data on the asset (see FontAsset), not a property.
   font: [],
+  // The file is intrinsic data (see SoundAsset); these two say how it is played by default, and a
+  // `Play sound` node scales the one and can add to the other.
+  sound: [
+    { key: SOUND_VOLUME_KEY, label: 'Volume', kind: 'number', default: 1, min: 0, max: 1, step: 0.05 },
+    { key: SOUND_LOOP_KEY, label: 'Loop', kind: 'boolean', default: false },
+  ],
   // How many particles and for how long; what one of them does is the asset's graph, whose Property
   // nodes add their rows after these – see `schemaForBag`.
   particles: [

src/core/properties/kinds.ts

@@ -106,6 +106,20 @@ export const KINDS = [
     field: true,
     port: true,
   },
+  /** A sound asset's id – a text, so it is tagged in a list like a texture. */
+  {
+    kind: 'sound',
+    label: 'Sound',
+    mark: '♪',
+    travels: 'string',
+    empty: '',
+    coerce: text,
+    tagged: true,
+    reference: true,
+    variable: true,
+    field: true,
+    port: true,
+  },
   { kind: 'object', label: 'Object', mark: '◇', travels: 'object', port: true },
   { kind: 'list', label: 'List', mark: '≡', travels: 'list', variable: true, port: true },
   { kind: 'struct', label: 'Struct', mark: '⊞', travels: 'struct', variable: true, port: true },

src/core/properties/sounds.ts

@@ -0,0 +1,15 @@
+import { booleanOf, type PropertyBag } from '../model/Properties';
+import { coerceAs } from './kinds';
+
+/** How loud this sound is by itself, 0–1; a `Play sound` node scales this rather than replacing it. */
+export const VOLUME_KEY = 'volume';
+/** Whether this sound is music: it repeats until something stops it. */
+export const LOOP_KEY = 'loop';
+
+export function volumeOf(bag: PropertyBag): number {
+  return coerceAs(bag[VOLUME_KEY], 'number', 1);
+}
+
+export function loopsOf(bag: PropertyBag): boolean {
+  return booleanOf(bag[LOOP_KEY]);
+}

src/core/store/HttpSoundStore.ts

@@ -0,0 +1,30 @@
+import { apiRequest } from './http';
+import type { SoundStore, StoredSound, UploadedSound } from './SoundStore';
+
+export class HttpSoundStore implements SoundStore {
+  constructor(private readonly endpoint: string) {}
+
+  async upload(bytes: Blob | ArrayBuffer, type: string, name = ''): Promise<UploadedSound> {
+    return this.request<UploadedSound>('POST', 'sounds', { bytes, type, name });
+  }
+
+  async list(): Promise<StoredSound[]> {
+    return this.request<StoredSound[]>('GET', 'sounds');
+  }
+
+  async delete(file: string): Promise<void> {
+    await this.request<void>('DELETE', `sounds/${encodeURIComponent(file)}`);
+  }
+
+  private async request<T>(
+    method: string,
+    route: string,
+    payload?: { bytes: Blob | ArrayBuffer; type: string; name: string },
+  ): Promise<T> {
+    return apiRequest<T>(this.endpoint, method, route, {
+      headers: payload ? { 'Content-Type': payload.type } : {},
+      body: payload?.bytes,
+      query: payload ? { name: payload.name } : undefined,
+    });
+  }
+}

src/core/store/SoundStore.ts

@@ -0,0 +1,27 @@
+export interface UploadedSound {
+  /** Content-addressed name in `data/sounds/`: `{sha256}.{mp3|ogg|wav|m4a}` (see `isSoundFile`). */
+  file: string;
+  /** The name the file was uploaded under, '' when the server knows none. Never a path. */
+  name: string;
+  bytes: number;
+}
+
+/** A sound the server holds, whether any project references it or not. */
+export interface StoredSound {
+  file: string;
+  name: string;
+  bytes: number;
+  updatedAt: string;
+}
+
+/**
+ * Raw bytes, like a font upload: the server derives name and format from the file itself. A sound
+ * is never re-encoded on the way, so what the server stores is what was picked – and how long it
+ * plays is nowhere in here, because only a decoder knows that.
+ */
+export interface SoundStore {
+  /** `name` is kept beside the file as its caption; the stored file is always the hash. */
+  upload(bytes: Blob | ArrayBuffer, type: string, name?: string): Promise<UploadedSound>;
+  list(): Promise<StoredSound[]>;
+  delete(file: string): Promise<void>;
+}

src/core/store/serialize.ts

@@ -17,6 +17,7 @@ import { isShaderNodeDef } from '../shader/nodes';
 import type { Asset, StateMachineAsset, TextureFrame } from '../model/Asset';
 import { isAssetType } from '../model/AssetTypes';
 import { isFontFile } from '../model/font';
+import { isSoundFile } from '../model/sound';
 import { emptyEdit, type InstanceEdit, isEmptyEdit } from '../model/instance';
 import {
   type FieldValue,
@@ -153,6 +154,9 @@ export function deserializeProject(raw: unknown, fallbackId = ''): Project {
     } else if (type === 'font') {
       const file = typeof a.file === 'string' && isFontFile(a.file) ? a.file : '';
       assets.push({ id, type, name, props, file });
+    } else if (type === 'sound') {
+      const file = typeof a.file === 'string' && isSoundFile(a.file) ? a.file : '';
+      assets.push({ id, type, name, props, file });
     } else if (type === 'animation') {
       // Kept even when they dangle, like a sprite's `texture`: a frame reads as no picture.
       const frames = Array.isArray(a.frames) ? a.frames : [];