From 5671a380443896e328da017898ae9d95994cb7ff Mon Sep 17 00:00:00 2001 From: MelekTaus Date: Tue, 15 Sep 2026 07:54:28 +0200 Subject: [PATCH] A game that stays silent is only half a game, so a sound is an asset like any other (#45) Sounds follow the road fonts already paved: the file goes to the server content-addressed under its sha256, the project keeps a named asset that points at it, and nothing is ever re-encoded. The format is read from the file's own signature, not from what the browser claims - the same .m4a arrives as audio/mp4, audio/x-m4a or as nothing at all. Playing is two nodes. "Play sound" takes the asset and may raise or lower its volume; every call is its own voice, so two jumps in a row are two jumps heard. Its loop input is a choice of three - from the asset, on, off - because a checkbox has to spend one of its two positions on "not set" and would lose a case: an asset that loops could then never be played once. Below the graph core stays browser-free: NodeContext hands out an asset id, BehaviorSystem resolves it against the project and folds the asset's own volume and loop into the call, and only viewport/SoundLibrary knows about Web Audio. A run without a sink plays nothing instead of throwing. The CI browser is muted, so the ear is no witness: the debug hook counts started and stopped voices, and step 34 reads that counter. --- .gitignore | 1 + api/index.php | 49 +++- api/lib/SoundStore.php | 157 ++++++++++++ src/core/commands/assetCommands.ts | 30 ++- src/core/graph/NodeContext.ts | 10 + src/core/graph/nodes/action/PlaySound.ts | 54 ++++ src/core/graph/nodes/action/StopSound.ts | 19 ++ src/core/graph/nodes/data/literals.ts | 5 + src/core/graph/nodes/index.ts | 6 + src/core/graph/nodes/shared.ts | 11 + src/core/model/Asset.ts | 11 + src/core/model/AssetTypes.ts | 8 +- src/core/model/sound.ts | 20 ++ src/core/properties/PropertySchema.ts | 7 + src/core/properties/kinds.ts | 14 + src/core/properties/sounds.ts | 15 ++ src/core/store/HttpSoundStore.ts | 30 +++ src/core/store/SoundStore.ts | 27 ++ src/core/store/serialize.ts | 4 + src/editor/App.ts | 7 +- src/editor/Editor.ts | 6 +- src/editor/EditorRun.ts | 6 +- src/editor/PlaySession.ts | 9 +- src/editor/debugHook.ts | 18 ++ src/editor/fileImport.ts | 53 ++-- src/editor/inspector/assetSection.ts | 6 + src/editor/inspector/fieldFactory.ts | 6 + src/editor/panels/AssetPanel.ts | 18 +- src/editor/sound/SoundPicker.ts | 310 +++++++++++++++++++++++ src/editor/sound/SoundPreview.ts | 44 ++++ src/editor/soundImport.ts | 105 ++++++++ src/runtime/systems/Audio.ts | 19 ++ src/runtime/systems/BehaviorSystem.ts | 31 ++- src/viewport/SoundLibrary.ts | 171 +++++++++++++ tests/core/node-catalog.test.ts | 47 ++++ tests/core/serialize.test.ts | 5 +- tests/core/sounds.test.ts | 231 +++++++++++++++++ tests/helpers.ts | 12 +- tests/viewport/sound-library.test.ts | 160 ++++++++++++ tools/e2e/21-list.mjs | 4 +- tools/e2e/34-sound.mjs | 275 ++++++++++++++++++++ tools/e2e/index.mjs | 2 + tools/e2e/lib/scaffold.mjs | 1 + tools/lib/audio.mjs | 47 ++++ tools/lib/browser.mjs | 4 + tools/lib/projects.mjs | 21 ++ tools/seed-e2e.mjs | 12 +- 47 files changed, 2066 insertions(+), 42 deletions(-) create mode 100644 api/lib/SoundStore.php create mode 100644 src/core/graph/nodes/action/PlaySound.ts create mode 100644 src/core/graph/nodes/action/StopSound.ts create mode 100644 src/core/model/sound.ts create mode 100644 src/core/properties/sounds.ts create mode 100644 src/core/store/HttpSoundStore.ts create mode 100644 src/core/store/SoundStore.ts create mode 100644 src/editor/sound/SoundPicker.ts create mode 100644 src/editor/sound/SoundPreview.ts create mode 100644 src/editor/soundImport.ts create mode 100644 src/runtime/systems/Audio.ts create mode 100644 src/viewport/SoundLibrary.ts create mode 100644 tests/core/sounds.test.ts create mode 100644 tests/viewport/sound-library.test.ts create mode 100644 tools/e2e/34-sound.mjs create mode 100644 tools/lib/audio.mjs diff --git a/.gitignore b/.gitignore index 666621d4..ca3f8450 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /data/projects/*.tmp /data/textures/ /data/fonts/ +/data/sounds/ /dist/ /node_modules/ /logs/ diff --git a/api/index.php b/api/index.php index 58513288..4f849054 100644 --- a/api/index.php +++ b/api/index.php @@ -21,20 +21,26 @@ declare(strict_types=1); * POST ?r=fonts&name=x.ttf raw font bytes → {file,name,bytes} * DELETE ?r=fonts/{file} → 204 * + * GET ?r=sounds → [{file,name,bytes,updatedAt}] + * POST ?r=sounds&name=x.wav raw sound bytes → {file,name,bytes} + * DELETE ?r=sounds/{file} → 204 + * * `name` is the file's original name, kept beside it as `{file}.json` (see OriginalName) and * empty when none is known. It is a caption, never a path: the stored file is always `{hash}.{ext}`. * - * Stored images and fonts are served straight from `data/textures/` and `data/fonts/` as - * static files; the API only writes them. + * Stored images, fonts and sounds are served straight from `data/textures/`, `data/fonts/` and + * `data/sounds/` as static files; the API only writes them. */ require __DIR__ . '/lib/FontStore.php'; require __DIR__ . '/lib/OriginalName.php'; require __DIR__ . '/lib/Response.php'; +require __DIR__ . '/lib/SoundStore.php'; require __DIR__ . '/lib/Store.php'; require __DIR__ . '/lib/TextureStore.php'; -const MAX_BODY_BYTES = 8 * 1024 * 1024; +/** Has to clear the largest single upload below, which is a sound; the web server has its own say. */ +const MAX_BODY_BYTES = 20 * 1024 * 1024; /** Upload ceiling for a single image. The editor downscales larger pictures before sending. */ const MAX_TEXTURE_BYTES = 4 * 1024 * 1024; @@ -42,6 +48,9 @@ const MAX_TEXTURE_BYTES = 4 * 1024 * 1024; /** Upload ceiling for a single font. A font cannot be shrunk on the way, so this is a plain limit. */ const MAX_FONT_BYTES = 4 * 1024 * 1024; +/** The same for a sound, four times as high: one music track is worth more than four typefaces. */ +const MAX_SOUND_BYTES = 16 * 1024 * 1024; + set_error_handler(static function (int $severity, string $message, string $file, int $line): bool { throw new ErrorException($message, 0, $severity, $file, $line); }); @@ -171,6 +180,39 @@ function routeFonts(array $parts, string $method): never Response::error('Method not allowed', 405); } +/** @param list $parts */ +function routeSounds(array $parts, string $method): never +{ + $store = new SoundStore(dirname(__DIR__) . '/data/sounds'); + + if (count($parts) === 1) { + if ($method === 'GET') { + Response::json($store->list()); + } + if ($method === 'POST') { + $bytes = Response::rawBody(); + if (strlen($bytes) > MAX_SOUND_BYTES) { + Response::error('Sound must be at most ' . (MAX_SOUND_BYTES / 1024 / 1024) . ' MB', 413); + } + try { + Response::json($store->store($bytes, (string) ($_GET['name'] ?? '')), 201); + } catch (InvalidSoundException $e) { + Response::error($e->getMessage(), 422); + } + } + Response::error('Method not allowed', 405); + } + + if ($method === 'DELETE') { + if (!$store->delete($parts[1])) { + Response::error('Sound not found', 404); + } + Response::noContent(); + } + + Response::error('Method not allowed', 405); +} + try { if ((int) ($_SERVER['CONTENT_LENGTH'] ?? 0) > MAX_BODY_BYTES) { Response::error('Request body too large', 413); @@ -188,6 +230,7 @@ try { 'projects' => routeProjects($parts, $method), 'textures' => routeTextures($parts, $method), 'fonts' => routeFonts($parts, $method), + 'sounds' => routeSounds($parts, $method), default => Response::error('Not found', 404), }; } catch (Throwable $e) { diff --git a/api/lib/SoundStore.php b/api/lib/SoundStore.php new file mode 100644 index 00000000..4c27bf38 --- /dev/null +++ b/api/lib/SoundStore.php @@ -0,0 +1,157 @@ + 'ogg', + 'ID3' => 'mp3', + ]; + + public function __construct(private readonly string $dir) + { + if (!is_dir($this->dir) && !mkdir($this->dir, 0775, true) && !is_dir($this->dir)) { + throw new RuntimeException("Cannot create sound directory {$this->dir}"); + } + } + + /** A stored file name, as it appears in a project document. */ + public static function isValidName(string $name): bool + { + return (bool) preg_match('/^[a-f0-9]{64}\.(mp3|ogg|wav|m4a)$/', $name); + } + + /** + * Stores the raw bytes of a sound and returns its record. + * + * The format is read from the file's own signature, never from anything the client says – + * browsers report a dropped `.m4a` as `audio/mp4`, `audio/x-m4a` or nothing at all, and an + * `.ogg` as either `audio/ogg` or `video/ogg`. + * + * `$originalName` is kept beside the file as a caption (see OriginalName): the hash says + * nothing, and there is no tag in here that is worth trusting for a label. + * + * @return array{file:string,name:string,bytes:int} + * @throws InvalidSoundException when the bytes are not a supported sound + */ + public function store(string $bytes, string $originalName = ''): array + { + $extension = self::extensionOf($bytes); + if ($extension === '') { + throw new InvalidSoundException('Not a sound file – use MP3, OGG, WAV or M4A'); + } + + $file = hash('sha256', $bytes) . '.' . $extension; + $path = $this->dir . '/' . $file; + if (!is_file($path)) { + $this->writeFile($path, $bytes); + } + OriginalName::write($this->dir, $file, $originalName); + + return ['file' => $file, 'name' => OriginalName::read($this->dir, $file), 'bytes' => strlen($bytes)]; + } + + /** + * Every stored file, newest first – what the sound picker offers. A sound has nothing to show, + * so the picker plays it instead; length is not read here, since only a decoder knows it. + * + * @return list + */ + public function list(): array + { + $items = []; + foreach (glob($this->dir . '/*.*') ?: [] as $path) { + $file = basename($path); + // Sidecars share the directory and are skipped: `{hash}.mp3.json` is not a valid name. + if (!self::isValidName($file)) { + continue; + } + $items[] = [ + 'file' => $file, + 'name' => OriginalName::read($this->dir, $file), + 'bytes' => (int) filesize($path), + 'updatedAt' => gmdate('c', (int) filemtime($path)), + ]; + } + usort($items, static fn(array $a, array $b): int => strcmp($b['updatedAt'], $a['updatedAt'])); + return $items; + } + + public function delete(string $file): bool + { + if (!self::isValidName($file)) { + return false; + } + $path = $this->dir . '/' . $file; + if (!is_file($path) || !unlink($path)) { + return false; + } + OriginalName::delete($this->dir, $file); + return true; + } + + /** The format these bytes are, `''` when they are none we take. */ + private static function extensionOf(string $bytes): string + { + // The shortest thing that can still carry a header of any of the four formats. + if (strlen($bytes) < 12) { + return ''; + } + foreach (self::SIGNATURES as $magic => $extension) { + if (str_starts_with($bytes, $magic)) { + return $extension; + } + } + // Both are RIFF/ISO container formats, so the type sits past the size field, not at byte 0. + if (str_starts_with($bytes, 'RIFF') && substr($bytes, 8, 4) === 'WAVE') { + return 'wav'; + } + if (substr($bytes, 4, 4) === 'ftyp') { + return 'm4a'; + } + return self::mp3Extension($bytes); + } + + /** + * A tagless MP3: a frame header is eleven set bits (`0xFF 0xEx`/`0xFx`), and the two bits below + * them name the MPEG version, of which `01` is reserved – demanding that much keeps a random + * binary that happens to start with `0xFF` out. + */ + private static function mp3Extension(string $bytes): string + { + $first = ord($bytes[0]); + $second = ord($bytes[1]); + $version = ($second >> 3) & 0b11; + return $first === 0xFF && ($second & 0xE0) === 0xE0 && $version !== 0b01 ? 'mp3' : ''; + } + + /** Temp file plus rename, so a reader never sees a half-written sound (see TextureStore). */ + private function writeFile(string $path, string $bytes): void + { + $tmp = $path . '.' . bin2hex(random_bytes(4)) . '.tmp'; + if (file_put_contents($tmp, $bytes, LOCK_EX) === false || !rename($tmp, $path)) { + @unlink($tmp); + throw new RuntimeException('Cannot write sound file'); + } + @chmod($path, 0664); + } +} + +/** Thrown by SoundStore::store when the uploaded bytes are not a usable sound. */ +final class InvalidSoundException extends RuntimeException +{ +} diff --git a/src/core/commands/assetCommands.ts b/src/core/commands/assetCommands.ts index a648b87e..c99cb249 100644 --- a/src/core/commands/assetCommands.ts +++ b/src/core/commands/assetCommands.ts @@ -1,8 +1,9 @@ import { createEmptyGraph } from '../graph/Graph'; import { newId } from '../ids'; -import type { Asset, FontAsset, StateMachineAsset, TextureAsset } from '../model/Asset'; +import type { Asset, FontAsset, SoundAsset, StateMachineAsset, TextureAsset } from '../model/Asset'; import type { AssetType } from '../model/AssetTypes'; import { isFontFile } from '../model/font'; +import { isSoundFile } from '../model/sound'; import { removeSubtrees } from '../model/hierarchy'; import type { PropertyBag, PropertyValue } from '../model/Properties'; import { objectHolders, type Project } from '../model/Project'; @@ -44,6 +45,8 @@ export function buildAsset(type: AssetType, name: string, id = newId()): Asset { return { ...base, type: 'shader', graph: createEmptyGraph() }; case 'font': return { ...base, type: 'font', file: '' }; + case 'sound': + return { ...base, type: 'sound', file: '' }; case 'particles': return { ...base, type: 'particles', graph: createEmptyGraph() }; } @@ -156,6 +159,31 @@ export class CreateFontCommand implements Command { } } +export function buildSoundAsset(name: string, file: string, id = newId()): SoundAsset { + return { id, name, props: defaultsFor('sound'), type: 'sound', file }; +} + +/** The sound is uploaded before the command runs, as a typeface is for `CreateFontCommand`. */ +export class CreateSoundCommand implements Command { + readonly label: string; + readonly asset: SoundAsset; + + constructor(name: string, file: string) { + this.asset = buildSoundAsset(name, file); + this.label = `Import sound "${name}"`; + } + + execute(project: Project): void { + if (!isSoundFile(this.asset.file)) + throw new CommandError(`"${this.asset.file}" is not a stored sound file`); + project.assets.push(this.asset); + } + + undo(project: Project): void { + project.assets = removeById(project.assets, this.asset.id); + } +} + /** * Deletes an asset and, as a cascade, every scene object referencing it, together with the objects * parented below those – in container templates as well as in scenes. diff --git a/src/core/graph/NodeContext.ts b/src/core/graph/NodeContext.ts index 40768a7f..97eb6430 100644 --- a/src/core/graph/NodeContext.ts +++ b/src/core/graph/NodeContext.ts @@ -119,6 +119,16 @@ export interface NodeContext { */ moveAndSlide(target: EntityHandle, dx: number, dy: number): { blockedX: boolean; blockedY: boolean }; log(message: string): void; + /** + * Starts a sound asset, named by its id. `volume` multiplies the asset's own (1 leaves it alone); + * `loop` is the caller's answer to whether it repeats, and `undefined` leaves that to the asset, + * because the asset says what a sound *is* and the node only colours it. Every call is a voice of + * its own: two plays overlap, whether they loop or not, which is what makes one sound effect + * fired twice in a row audible twice. + */ + playSound(assetId: string, volume: number, loop: boolean | undefined): void; + /** Silences every voice of that sound; `''` silences everything this run started. */ + stopSound(assetId: string): void; /** * The position is local to `parent` (world when none); null when the asset does not exist or * cannot be placed. A spawned behavior runs its `On start` on the next tick. diff --git a/src/core/graph/nodes/action/PlaySound.ts b/src/core/graph/nodes/action/PlaySound.ts new file mode 100644 index 00000000..3b912a48 --- /dev/null +++ b/src/core/graph/nodes/action/PlaySound.ts @@ -0,0 +1,54 @@ +import type { NodeDef, PortDef } from '../../NodeRegistry'; +import { asNumber, asString, type Value } from '../../values'; +import { exec, num, soundPort, withDefault } from '../shared'; + +/** + * Three answers, not two: a checkbox cannot say "leave it to the asset", and without that a sound + * whose own `Loop` is on could never be played as a one-shot. + */ +const LOOP_OPTIONS = [ + { value: '', label: 'From the asset' }, + { value: 'true', label: 'On' }, + { value: 'false', label: 'Off' }, +]; + +/** `any`, not `boolean`, so a graph can still drive the choice – see `loopChoice`. */ +const loopPort = (): PortDef => ({ + key: 'loop', + label: 'loop', + type: 'any', + default: '', + field: { key: 'loop', label: 'Loop', kind: 'enum', default: '', options: LOOP_OPTIONS }, +}); + +/** + * What the node says about looping: nothing (the asset decides) or an answer that overrules it. + * A boolean arriving over an edge is taken literally, which is what wiring a comparison or a + * variable into this input is for. + */ +export function loopChoice(value: Value | undefined): boolean | undefined { + if (typeof value === 'boolean') return value; + const text = asString(value); + if (text === 'true') return true; + if (text === 'false') return false; + return undefined; +} + +export const PlaySound: NodeDef = { + type: 'PlaySound', + label: 'Play sound', + category: 'action', + description: + 'Plays a sound asset. "volume" multiplies the volume the asset itself carries and "loop" either ' + + "leaves the asset's own setting alone or overrules it, so the asset says what the sound is and " + + 'the node only colours it. Each play is a voice of its own: firing this twice in a row is heard ' + + 'twice, overlapping.', + inputs: [exec(), soundPort(), withDefault(num('volume'), 1), loopPort()], + outputs: [exec()], + params: [], + run: (ctx, inputs) => { + const sound = asString(inputs.sound); + if (sound) ctx.playSound(sound, asNumber(inputs.volume), loopChoice(inputs.loop)); + return { next: 'exec' }; + }, +}; diff --git a/src/core/graph/nodes/action/StopSound.ts b/src/core/graph/nodes/action/StopSound.ts new file mode 100644 index 00000000..a8b32bd5 --- /dev/null +++ b/src/core/graph/nodes/action/StopSound.ts @@ -0,0 +1,19 @@ +import type { NodeDef } from '../../NodeRegistry'; +import { asString } from '../../values'; +import { exec, soundPort } from '../shared'; + +export const StopSound: NodeDef = { + type: 'StopSound', + label: 'Stop sound', + category: 'action', + description: + 'Silences every voice of a sound, however many plays started it – including a looping one. ' + + 'With no sound picked it silences everything the run is playing.', + inputs: [exec(), soundPort()], + outputs: [exec()], + params: [], + run: (ctx, inputs) => { + ctx.stopSound(asString(inputs.sound)); + return { next: 'exec' }; + }, +}; diff --git a/src/core/graph/nodes/data/literals.ts b/src/core/graph/nodes/data/literals.ts index f5ec686c..be3225f0 100644 --- a/src/core/graph/nodes/data/literals.ts +++ b/src/core/graph/nodes/data/literals.ts @@ -35,6 +35,11 @@ export const StateMachineLiteral = literal( 'statemachine', 'A constant state machine, for a "State branch" or a "Set state" to work on.', ); +export const SoundLiteral = literal( + 'Sound', + 'sound', + 'A constant sound, for a "Play sound" or a "Stop sound" to work on.', +); export const ListLiteral = literal( 'List', 'list', diff --git a/src/core/graph/nodes/index.ts b/src/core/graph/nodes/index.ts index b85aa5a5..d80034f5 100644 --- a/src/core/graph/nodes/index.ts +++ b/src/core/graph/nodes/index.ts @@ -3,10 +3,12 @@ import { Destroy } from './action/Destroy'; import { Emit } from './action/Emit'; import { Log } from './action/Log'; import { Move } from './action/Move'; +import { PlaySound } from './action/PlaySound'; import { RestartGame } from './action/RestartGame'; import { SetPosition } from './action/SetPosition'; import { Spawn } from './action/Spawn'; import { StopGame } from './action/StopGame'; +import { StopSound } from './action/StopSound'; import { Abs } from './data/Abs'; import { Add } from './data/Add'; import { And } from './data/And'; @@ -32,6 +34,7 @@ import { BooleanLiteral, ListLiteral, NumberLiteral, + SoundLiteral, StateMachineLiteral, StringLiteral, TextureLiteral, @@ -146,6 +149,7 @@ export const BUILT_IN_NODES: readonly NodeDef[] = [ StringLiteral, TextureLiteral, StateMachineLiteral, + SoundLiteral, Add, Subtract, Multiply, @@ -195,6 +199,8 @@ export const BUILT_IN_NODES: readonly NodeDef[] = [ Spawn, Destroy, Emit, + PlaySound, + StopSound, Log, StopGame, RestartGame, diff --git a/src/core/graph/nodes/shared.ts b/src/core/graph/nodes/shared.ts index 1e3bcf18..3b0cdf33 100644 --- a/src/core/graph/nodes/shared.ts +++ b/src/core/graph/nodes/shared.ts @@ -36,6 +36,17 @@ export const keyParam: PropertyDef = { key: 'key', label: 'Key', kind: 'key', de /** No `assetTypes`: any placeable asset is a valid answer here. */ export const assetParam: PropertyDef = { key: 'asset', label: 'Asset', kind: 'asset', default: '' }; +/** + * The `sound` input of the two sound nodes; unconnected, it shows the sound picker – the way + * `machinePort` shows the machine picker. A `Sound` literal is what feeds it through an edge. + */ +export const soundPort = (): PortDef => ({ + key: 'sound', + label: 'sound', + type: 'string', + field: { key: 'sound', label: 'Sound', kind: 'sound', default: '' }, +}); + export function binaryMath(type: string, description: string, fn: (a: number, b: number) => number): NodeDef { return { type, diff --git a/src/core/model/Asset.ts b/src/core/model/Asset.ts index 4f51f03c..57c86203 100644 --- a/src/core/model/Asset.ts +++ b/src/core/model/Asset.ts @@ -129,6 +129,16 @@ export interface FontAsset extends AssetBase { file: string; } +/** + * `file` names the sound in `data/sounds/` the way a font names its typeface: content-addressed + * `{sha256}.{ext}`, empty when the upload failed. Nothing wears a sound – a graph plays it with + * `Play sound`, and `props` says how loud and whether it repeats (see `properties/sounds.ts`). + */ +export interface SoundAsset extends AssetBase { + type: 'sound'; + file: string; +} + /** * The graph is the motion of one particle: every output of its `Output` node is a function of the * particle's age and its own random number (see `core/particles/compile.ts`). A placed object is @@ -153,6 +163,7 @@ export type Asset = | StateMachineAsset | ShaderAsset | FontAsset + | SoundAsset | ParticlesAsset; /** The assets edited as a node graph – what the graph canvas and the graph commands work on. */ diff --git a/src/core/model/AssetTypes.ts b/src/core/model/AssetTypes.ts index cd7fcb8a..3cc5efe8 100644 --- a/src/core/model/AssetTypes.ts +++ b/src/core/model/AssetTypes.ts @@ -12,6 +12,7 @@ export const ASSET_TYPES = [ 'statemachine', 'shader', 'font', + 'sound', 'particles', ] as const; @@ -31,17 +32,19 @@ export const ASSET_TYPE_LABELS: Record = { statemachine: 'State machine', shader: 'Shader', font: 'Font', + sound: 'Sound', particles: 'Particles', }; /** * Textures and animations are not placeable: they are pictures a sprite wears through `texture`, * as a shader is worn through `shader` and a font through `font`. A state machine is not either: - * a behavior refers to it by id, and a Variables asset is declarations a graph reads by id. + * a behavior refers to it by id, and a Variables asset is declarations a graph reads by id. A + * sound is nowhere on screen at all – a graph plays it, so there is nothing to place. */ export type PlaceableAssetType = Exclude< AssetType, - 'scene' | 'texture' | 'animation' | 'statemachine' | 'shader' | 'font' | 'variables' + 'scene' | 'texture' | 'animation' | 'statemachine' | 'shader' | 'font' | 'sound' | 'variables' >; const NOT_PLACEABLE: ReadonlySet = new Set([ @@ -51,6 +54,7 @@ const NOT_PLACEABLE: ReadonlySet = new Set([ 'statemachine', 'shader', 'font', + 'sound', 'variables', ]); diff --git a/src/core/model/sound.ts b/src/core/model/sound.ts new file mode 100644 index 00000000..6d806dce --- /dev/null +++ b/src/core/model/sound.ts @@ -0,0 +1,20 @@ +import type { SoundAsset } from './Asset'; +import { findAsset, type Project } from './Project'; + +/** Mirrors `SoundStore::isValidName` on the server, where the name is derived from the bytes. */ +const FILE_PATTERN = /^[a-f0-9]{64}\.(mp3|ogg|wav|m4a)$/; + +export function isSoundFile(name: string): boolean { + return FILE_PATTERN.test(name); +} + +/** The file a `sound` value points at, empty when the id is unset or no longer resolves. */ +export function soundFileOf(project: Project, id: string): string { + if (id === '') return ''; + const asset = findAsset(project, id); + return asset?.type === 'sound' ? asset.file : ''; +} + +export function isSoundAsset(asset: { type: string }): asset is SoundAsset { + return asset.type === 'sound'; +} diff --git a/src/core/properties/PropertySchema.ts b/src/core/properties/PropertySchema.ts index d9d138ce..e06d5b0d 100644 --- a/src/core/properties/PropertySchema.ts +++ b/src/core/properties/PropertySchema.ts @@ -40,6 +40,7 @@ import { TIME_SCALE_KEY, } from './particles'; import { SHADER_KEY, shaderOf, shaderParamDefs } from './shaders'; +import { LOOP_KEY as SOUND_LOOP_KEY, VOLUME_KEY as SOUND_VOLUME_KEY } from './sounds'; /** * What a kind's definition carries beyond key, label and default. `decimals` belongs only on @@ -222,6 +223,12 @@ const ASSET_SCHEMAS: Record = { shader: [], // The typeface itself is intrinsic data on the asset (see FontAsset), not a property. font: [], + // The file is intrinsic data (see SoundAsset); these two say how it is played by default, and a + // `Play sound` node scales the one and can add to the other. + sound: [ + { key: SOUND_VOLUME_KEY, label: 'Volume', kind: 'number', default: 1, min: 0, max: 1, step: 0.05 }, + { key: SOUND_LOOP_KEY, label: 'Loop', kind: 'boolean', default: false }, + ], // How many particles and for how long; what one of them does is the asset's graph, whose Property // nodes add their rows after these – see `schemaForBag`. particles: [ diff --git a/src/core/properties/kinds.ts b/src/core/properties/kinds.ts index 128b0864..5a3d164c 100644 --- a/src/core/properties/kinds.ts +++ b/src/core/properties/kinds.ts @@ -106,6 +106,20 @@ export const KINDS = [ field: true, port: true, }, + /** A sound asset's id – a text, so it is tagged in a list like a texture. */ + { + kind: 'sound', + label: 'Sound', + mark: '♪', + travels: 'string', + empty: '', + coerce: text, + tagged: true, + reference: true, + variable: true, + field: true, + port: true, + }, { kind: 'object', label: 'Object', mark: '◇', travels: 'object', port: true }, { kind: 'list', label: 'List', mark: '≡', travels: 'list', variable: true, port: true }, { kind: 'struct', label: 'Struct', mark: '⊞', travels: 'struct', variable: true, port: true }, diff --git a/src/core/properties/sounds.ts b/src/core/properties/sounds.ts new file mode 100644 index 00000000..28d2910c --- /dev/null +++ b/src/core/properties/sounds.ts @@ -0,0 +1,15 @@ +import { booleanOf, type PropertyBag } from '../model/Properties'; +import { coerceAs } from './kinds'; + +/** How loud this sound is by itself, 0–1; a `Play sound` node scales this rather than replacing it. */ +export const VOLUME_KEY = 'volume'; +/** Whether this sound is music: it repeats until something stops it. */ +export const LOOP_KEY = 'loop'; + +export function volumeOf(bag: PropertyBag): number { + return coerceAs(bag[VOLUME_KEY], 'number', 1); +} + +export function loopsOf(bag: PropertyBag): boolean { + return booleanOf(bag[LOOP_KEY]); +} diff --git a/src/core/store/HttpSoundStore.ts b/src/core/store/HttpSoundStore.ts new file mode 100644 index 00000000..cb622be3 --- /dev/null +++ b/src/core/store/HttpSoundStore.ts @@ -0,0 +1,30 @@ +import { apiRequest } from './http'; +import type { SoundStore, StoredSound, UploadedSound } from './SoundStore'; + +export class HttpSoundStore implements SoundStore { + constructor(private readonly endpoint: string) {} + + async upload(bytes: Blob | ArrayBuffer, type: string, name = ''): Promise { + return this.request('POST', 'sounds', { bytes, type, name }); + } + + async list(): Promise { + return this.request('GET', 'sounds'); + } + + async delete(file: string): Promise { + await this.request('DELETE', `sounds/${encodeURIComponent(file)}`); + } + + private async request( + method: string, + route: string, + payload?: { bytes: Blob | ArrayBuffer; type: string; name: string }, + ): Promise { + return apiRequest(this.endpoint, method, route, { + headers: payload ? { 'Content-Type': payload.type } : {}, + body: payload?.bytes, + query: payload ? { name: payload.name } : undefined, + }); + } +} diff --git a/src/core/store/SoundStore.ts b/src/core/store/SoundStore.ts new file mode 100644 index 00000000..916d0b60 --- /dev/null +++ b/src/core/store/SoundStore.ts @@ -0,0 +1,27 @@ +export interface UploadedSound { + /** Content-addressed name in `data/sounds/`: `{sha256}.{mp3|ogg|wav|m4a}` (see `isSoundFile`). */ + file: string; + /** The name the file was uploaded under, '' when the server knows none. Never a path. */ + name: string; + bytes: number; +} + +/** A sound the server holds, whether any project references it or not. */ +export interface StoredSound { + file: string; + name: string; + bytes: number; + updatedAt: string; +} + +/** + * Raw bytes, like a font upload: the server derives name and format from the file itself. A sound + * is never re-encoded on the way, so what the server stores is what was picked – and how long it + * plays is nowhere in here, because only a decoder knows that. + */ +export interface SoundStore { + /** `name` is kept beside the file as its caption; the stored file is always the hash. */ + upload(bytes: Blob | ArrayBuffer, type: string, name?: string): Promise; + list(): Promise; + delete(file: string): Promise; +} diff --git a/src/core/store/serialize.ts b/src/core/store/serialize.ts index 33f419af..d9732311 100644 --- a/src/core/store/serialize.ts +++ b/src/core/store/serialize.ts @@ -17,6 +17,7 @@ import { isShaderNodeDef } from '../shader/nodes'; import type { Asset, StateMachineAsset, TextureFrame } from '../model/Asset'; import { isAssetType } from '../model/AssetTypes'; import { isFontFile } from '../model/font'; +import { isSoundFile } from '../model/sound'; import { emptyEdit, type InstanceEdit, isEmptyEdit } from '../model/instance'; import { type FieldValue, @@ -153,6 +154,9 @@ export function deserializeProject(raw: unknown, fallbackId = ''): Project { } else if (type === 'font') { const file = typeof a.file === 'string' && isFontFile(a.file) ? a.file : ''; assets.push({ id, type, name, props, file }); + } else if (type === 'sound') { + const file = typeof a.file === 'string' && isSoundFile(a.file) ? a.file : ''; + assets.push({ id, type, name, props, file }); } else if (type === 'animation') { // Kept even when they dangle, like a sprite's `texture`: a frame reads as no picture. const frames = Array.isArray(a.frames) ? a.frames : []; diff --git a/src/editor/App.ts b/src/editor/App.ts index 48409332..f14fcec6 100644 --- a/src/editor/App.ts +++ b/src/editor/App.ts @@ -1,9 +1,11 @@ import type { Application } from 'pixi.js'; import { HttpFontStore } from '../core/store/HttpFontStore'; import { HttpProjectStore } from '../core/store/HttpProjectStore'; +import { HttpSoundStore } from '../core/store/HttpSoundStore'; import { HttpTextureStore } from '../core/store/HttpTextureStore'; import { requestRender, type VStack } from '../ui'; import { fontLibrary } from '../viewport/FontLibrary'; +import { soundLibrary } from '../viewport/SoundLibrary'; import { textureLibrary } from '../viewport/TextureLibrary'; import { installDebugHook } from './debugHook'; import { Editor } from './Editor'; @@ -13,16 +15,18 @@ const API_ENDPOINT = '/game-maker/api/index.php'; /** * Where uploaded images live: the API writes them, the browser reads them straight off disk as - * static files named by their content hash. Fonts work the same way, one directory over. + * static files named by their content hash. Fonts and sounds work the same way, one directory over. */ const TEXTURE_BASE = '/game-maker/data/textures'; const FONT_BASE = '/game-maker/data/fonts'; +const SOUND_BASE = '/game-maker/data/sounds'; /** Wires state, stores and the editor composition together and opens the first project. */ export async function startEditor(app: Application, root: VStack): Promise { const state = new EditorState(); textureLibrary.configure(TEXTURE_BASE); fontLibrary.configure(FONT_BASE); + soundLibrary.configure(SOUND_BASE); const editor = new Editor( app, root, @@ -30,6 +34,7 @@ export async function startEditor(app: Application, root: VStack): Promise new HttpProjectStore(API_ENDPOINT), new HttpTextureStore(API_ENDPOINT), new HttpFontStore(API_ENDPOINT), + new HttpSoundStore(API_ENDPOINT), ); // An image or a font arrives long after the click that asked for it, and the widget waiting for // it repaints without telling anyone: both libraries are a wake-up of their own. diff --git a/src/editor/Editor.ts b/src/editor/Editor.ts index 2874c924..920ba366 100644 --- a/src/editor/Editor.ts +++ b/src/editor/Editor.ts @@ -3,6 +3,7 @@ import type { AssetType } from '../core/model/AssetTypes'; import { findAsset, findObjectHolder } from '../core/model/Project'; import type { ProjectStore } from '../core/store/ProjectStore'; import type { FontStore } from '../core/store/FontStore'; +import type { SoundStore } from '../core/store/SoundStore'; import type { TextureStore } from '../core/store/TextureStore'; import { gameClock } from '../core/time'; import { @@ -151,6 +152,7 @@ export class Editor implements PlayHost { store: ProjectStore, textures: TextureStore, fonts: FontStore, + sounds: SoundStore, ) { this.viewport = new SceneViewport(state); this.gameView = new GameViewport(state); @@ -196,9 +198,9 @@ export class Editor implements PlayHost { this.workspace = new HStack(); this.leftColumn = new VStack({ gap: 1 }); - this.assets = new AssetPanel(state, textures, fonts); + this.assets = new AssetPanel(state, textures, fonts, sounds); this.leftColumn.add(this.assets, 1); - installFileImport(app.canvas, { state, textures, fonts }); + installFileImport(app.canvas, { state, textures, fonts, sounds }); this.leftColumn.add(this.hierarchy, 1); this.centre = new VStack(); diff --git a/src/editor/EditorRun.ts b/src/editor/EditorRun.ts index 962983e0..f2ff790c 100644 --- a/src/editor/EditorRun.ts +++ b/src/editor/EditorRun.ts @@ -5,6 +5,7 @@ import { hasEditorRun, runsInEditor } from '../core/properties/editorRun'; import { addGameSystems, type BehaviorSystem, behaviorError } from '../runtime/systems/BehaviorSystem'; import { Input } from '../runtime/systems/Input'; import { type ModelWrite, World } from '../runtime/World'; +import { soundLibrary } from '../viewport/SoundLibrary'; import type { LiveMachineState } from './graph/GraphCanvas'; import type { EditorState } from './state'; @@ -119,7 +120,7 @@ export class EditorRun { // Baking a container's content into the scene would be an edit nobody asked for, so it is off // here: the content runs, but resolved from its template (see `WorldOptions.bakeContainers`). const fresh = World.instantiate(project, scene.id, { bakeContainers: false }); - this.behaviors = addGameSystems(fresh, project, this.input, (e) => runsInEditor(e.props)); + this.behaviors = addGameSystems(fresh, project, this.input, (e) => runsInEditor(e.props), soundLibrary); this.world = fresh; this.hadFocus = this.hooks.focused(); this.input.attach(this.hooks.focused); @@ -153,6 +154,9 @@ export class EditorRun { private stop(): void { if (!this.world) return; + // Its sounds end with it, for the reason a played run's do – an editor run is restarted by any + // edit, and a loop left behind would pile up one voice per edit. + soundLibrary.stopAll(); this.world.destroy(); this.world = null; this.behaviors = null; diff --git a/src/editor/PlaySession.ts b/src/editor/PlaySession.ts index f42eef20..c08b5e13 100644 --- a/src/editor/PlaySession.ts +++ b/src/editor/PlaySession.ts @@ -16,6 +16,7 @@ import { gameClock } from '../core/time'; import { Input } from '../runtime/systems/Input'; import { World } from '../runtime/World'; import { ui } from '../ui'; +import { soundLibrary } from '../viewport/SoundLibrary'; import type { GameViewport } from './viewport/GameViewport'; import type { GraphEditor } from './graph/GraphEditor'; import type { LiveMachineState } from './graph/GraphCanvas'; @@ -72,6 +73,9 @@ export class PlaySession { togglePause(): void { if (!this.world) return; this.state.setPaused(!this.state.paused); + // A held note is part of the picture a pause freezes: the sounds stop where they are and carry + // on mid-note, rather than playing to their end over a still image. + soundLibrary.setSuspended(this.state.paused); ui().focus.blur(); this.state.setNotice(this.playNotice(), 'quiet'); } @@ -109,7 +113,7 @@ export class PlaySession { const world = World.instantiate(runtime, scene.id); this.world = world; this.input.attach(); - this.behaviors = addGameSystems(world, runtime, this.input); + this.behaviors = addGameSystems(world, runtime, this.input, undefined, soundLibrary); this.host.gameView.setWorld(world); this.host.showGame(); this.state.setNotice(this.playNotice(), 'quiet'); @@ -126,6 +130,9 @@ export class PlaySession { stop(): void { // The run's seconds end with it: the editor goes back to the picture of a freshly loaded scene. gameClock.reset(); + // And its sounds end with it, looping music included – nothing may outlive the world. + soundLibrary.setSuspended(false); + soundLibrary.stopAll(); const world = this.world; this.world = null; this.behaviors = null; diff --git a/src/editor/debugHook.ts b/src/editor/debugHook.ts index 41f45425..809ecca8 100644 --- a/src/editor/debugHook.ts +++ b/src/editor/debugHook.ts @@ -7,9 +7,11 @@ import { effectiveScale, gameClock } from '../core/time'; import { activeDrag, Button, isRendering, renderCount, requestRender, ui, Widget } from '../ui'; import { particleLibrary } from '../viewport/ParticleLibrary'; import { shaderLibrary } from '../viewport/ShaderLibrary'; +import { soundLibrary } from '../viewport/SoundLibrary'; import { textureLibrary } from '../viewport/TextureLibrary'; import type { CentreTab, Editor } from './Editor'; import { currentFontPicker } from './font/FontPicker'; +import { currentSoundPicker } from './sound/SoundPicker'; import { currentImagePicker } from './texture/ImagePicker'; import type { EditorState } from './state'; @@ -361,6 +363,22 @@ export function installDebugHook(app: Application, state: EditorState, editor: E /** The same for the font picker, whose rows are the preview. */ fontPicker: () => currentFontPicker()?.debugInfo() ?? null, fontPickerRow: (file: string) => currentFontPicker()?.rowGlobal(file) ?? null, + /** And for the sound picker, whose rows carry a ▶ of their own. */ + soundPicker: () => currentSoundPicker()?.debugInfo() ?? null, + soundPickerRow: (file: string) => currentSoundPicker()?.rowGlobal(file) ?? null, + soundPickerPlay: (file: string) => currentSoundPicker()?.playGlobal(file) ?? null, + /** + * Voices started, stopped and in flight. CI runs the browser with `--mute-audio`, so nothing + * is ever heard there: these counters are the only way a check can see that a graph played + * anything at all, the way `renderStats` is for a repaint that did not happen. + */ + sound: { + stats: () => soundLibrary.stats(), + duration: (file: string) => soundLibrary.duration(file), + /** Decodes ahead of a check, so a play can be asserted without racing the network. */ + load: (file: string) => soundLibrary.load(file).then((buffer) => buffer?.duration ?? null), + stopAll: () => soundLibrary.stopAll(), + }, /** Which typeface a text ended up drawn in, and how wide it made it. */ textInfo: (id: string) => viewport.textInfo(id), gameTextInfo: (id: string) => gameView.textInfo(id), diff --git a/src/editor/fileImport.ts b/src/editor/fileImport.ts index 1f10ea83..4e1a77fc 100644 --- a/src/editor/fileImport.ts +++ b/src/editor/fileImport.ts @@ -1,13 +1,14 @@ import { type FontImportDeps, importFonts, isFontFileName } from './fontImport'; +import { importSounds, isSoundFileName, type SoundImportDeps } from './soundImport'; import { importImages, type TextureImportDeps } from './textureImport'; -export type FileImportDeps = TextureImportDeps & FontImportDeps; +export type FileImportDeps = TextureImportDeps & FontImportDeps & SoundImportDeps; /** * Wires drop-onto-`target` and paste, and returns the remover for both. Paste is bound to the * window because the canvas cannot hold focus, and skipped while a DOM text field has it. * - * Pictures and typefaces arrive the same way, so one pair of handlers sorts them apart. + * Pictures, typefaces and sounds arrive the same way, so one pair of handlers sorts them apart. */ export function installFileImport(target: HTMLElement, deps: FileImportDeps): () => void { const onDragOver = (event: DragEvent) => { @@ -17,24 +18,22 @@ export function installFileImport(target: HTMLElement, deps: FileImportDeps): () if (event.dataTransfer) event.dataTransfer.dropEffect = 'copy'; }; const onDrop = (event: DragEvent) => { - const images = imageFilesOf(event.dataTransfer); - const fonts = fontFilesOf(event.dataTransfer); - if (images.length === 0 && fonts.length === 0) { + const files = sortFiles(event.dataTransfer); + if (files.none) { if (!event.dataTransfer?.files.length) return; event.preventDefault(); - deps.state.setNotice('Only image and font files can be dropped into the editor.'); + deps.state.setNotice('Only image, font and sound files can be dropped into the editor.'); return; } event.preventDefault(); - run(images, fonts, deps); + run(files, deps); }; const onPaste = (event: ClipboardEvent) => { if (isTextFieldFocused()) return; - const images = imageFilesOf(event.clipboardData); - const fonts = fontFilesOf(event.clipboardData); - if (images.length === 0 && fonts.length === 0) return; + const files = sortFiles(event.clipboardData); + if (files.none) return; event.preventDefault(); - run(images, fonts, deps); + run(files, deps); }; target.addEventListener('dragover', onDragOver); @@ -47,23 +46,31 @@ export function installFileImport(target: HTMLElement, deps: FileImportDeps): () }; } -/** Images first, then fonts, so a mixed drop ends on one notice instead of two racing ones. */ -function run(images: readonly File[], fonts: readonly File[], deps: FileImportDeps): void { +interface SortedFiles { + images: File[]; + fonts: File[]; + sounds: File[]; + none: boolean; +} + +function sortFiles(data: DataTransfer | null): SortedFiles { + const all = [...(data?.files ?? [])]; + const images = all.filter((file) => file.type.startsWith('image/')); + const fonts = all.filter((file) => isFontFileName(file.name)); + const sounds = all.filter((file) => isSoundFileName(file.name)); + return { images, fonts, sounds, none: images.length + fonts.length + sounds.length === 0 }; +} + +/** One kind after the other, so a mixed drop ends on one notice instead of three racing ones. */ +function run(files: SortedFiles, deps: FileImportDeps): void { const work = async () => { - if (images.length) await importImages(images, deps); - if (fonts.length) await importFonts(fonts, deps); + if (files.images.length) await importImages(files.images, deps); + if (files.fonts.length) await importFonts(files.fonts, deps); + if (files.sounds.length) await importSounds(files.sounds, deps); }; void work().catch((error: unknown) => deps.state.setNotice(String(error), 'error')); } -function imageFilesOf(data: DataTransfer | null): File[] { - return [...(data?.files ?? [])].filter((file) => file.type.startsWith('image/')); -} - -function fontFilesOf(data: DataTransfer | null): File[] { - return [...(data?.files ?? [])].filter((file) => isFontFileName(file.name)); -} - /** * During a drag the files themselves are not readable yet – only kind and type are, and a font * usually has no type at all. Anything that is not plainly text therefore counts as a candidate; diff --git a/src/editor/inspector/assetSection.ts b/src/editor/inspector/assetSection.ts index 6a3d8d39..961b22a1 100644 --- a/src/editor/inspector/assetSection.ts +++ b/src/editor/inspector/assetSection.ts @@ -24,6 +24,7 @@ import { markedAssetSchema } from '../../core/properties/deviation'; import { isModified, resolveAssetProps } from '../../core/properties/resolve'; import { Dropdown, HStack, Label, Menu, Theme } from '../../ui'; import { FontPreview } from '../font/FontPreview'; +import { SoundPreview } from '../sound/SoundPreview'; import { aspectActions, sizeCommand } from './aspectAction'; import { actionIf, createFieldRow, type FieldHandlers } from './fieldFactory'; import { addNameField, addNote, addTitle, type InspectorHost } from './host'; @@ -76,6 +77,11 @@ export function buildAssetSection(host: InspectorHost, assetId: string): void { host.body.add( asset.file ? new FontPreview(asset.file) : new Label('Not uploaded.', { style: 'warning' }), ); + // A sound has two rows above this; the preview is the only way to find out which sound it is. + if (asset.type === 'sound') + host.body.add( + asset.file ? new SoundPreview(asset.file) : new Label('Not uploaded.', { style: 'warning' }), + ); } /** Pushes the current asset values into the rows. False when the asset is gone (rebuild). */ diff --git a/src/editor/inspector/fieldFactory.ts b/src/editor/inspector/fieldFactory.ts index bc500e18..7220a990 100644 --- a/src/editor/inspector/fieldFactory.ts +++ b/src/editor/inspector/fieldFactory.ts @@ -237,6 +237,11 @@ const REFERENCES: Record = { choices: (project) => assetRefChoices(project, ['statemachine']).map((a) => ({ id: a.id, label: a.name })), }, + sound: { + none: NO_ASSET_LABEL, + label: assetRefLabel, + choices: (project) => assetRefChoices(project, ['sound']).map((a) => ({ id: a.id, label: a.name })), + }, }; /** An id in the words of its reference kind; nothing for a kind that names nothing. */ @@ -362,6 +367,7 @@ export function createField( case 'asset': case 'texture': case 'statemachine': + case 'sound': return referenceField(ctx, def, REFERENCES[def.kind], value, handlers); } } diff --git a/src/editor/panels/AssetPanel.ts b/src/editor/panels/AssetPanel.ts index 34fb60e0..db60c922 100644 --- a/src/editor/panels/AssetPanel.ts +++ b/src/editor/panels/AssetPanel.ts @@ -16,11 +16,13 @@ import { findAsset } from '../../core/model/Project'; import { spriteFromTexture } from '../../core/properties/aspect'; import { assetDeviates } from '../../core/properties/deviation'; import type { FontStore } from '../../core/store/FontStore'; +import type { SoundStore } from '../../core/store/SoundStore'; import type { TextureStore } from '../../core/store/TextureStore'; import { Button, ListView, Menu, type MenuItem, Panel, ScrollContainer, startDrag } from '../../ui'; import { confirmDeleteAsset } from '../actions'; import type { EditorState } from '../state'; import { openFontPicker } from '../font/FontPicker'; +import { openSoundPicker } from '../sound/SoundPicker'; import { openImagePicker } from '../texture/ImagePicker'; /** @@ -39,6 +41,8 @@ export class AssetPanel extends Panel { private readonly textures: TextureStore | null = null, /** The same for typefaces and the Font entry. */ private readonly fonts: FontStore | null = null, + /** And for sounds and the Sound entry. */ + private readonly sounds: SoundStore | null = null, ) { super({ title: 'Assets', padding: 0, gap: 0 }); this.minWidth = 160; @@ -100,12 +104,14 @@ export class AssetPanel extends Panel { } /** - * A texture is a picture and a font a file, so their menu entries offer what the server already - * holds instead of creating an empty asset. Uploading a new one is a button inside that dialog. + * A texture is a picture, a font and a sound are files, so their menu entries offer what the + * server already holds instead of creating an empty asset. Uploading a new one is a button + * inside that dialog. */ private addAsset(type: AssetType): void { if (type === 'texture') this.importTextures(); else if (type === 'font') this.importFonts(); + else if (type === 'sound') this.importSounds(); else this.createAsset(type); } @@ -125,6 +131,14 @@ export class AssetPanel extends Panel { openFontPicker({ state: this.state, fonts: this.fonts }); } + private importSounds(): void { + if (!this.sounds) { + this.state.setNotice('Sound upload is not available.'); + return; + } + openSoundPicker({ state: this.state, sounds: this.sounds }); + } + /** Double-click / Enter opens the asset's tab; a scene also becomes the active one. Placing is a drag. */ private activate(id: string): void { const asset = findAsset(this.state.project, id); diff --git a/src/editor/sound/SoundPicker.ts b/src/editor/sound/SoundPicker.ts new file mode 100644 index 00000000..8eb50702 --- /dev/null +++ b/src/editor/sound/SoundPicker.ts @@ -0,0 +1,310 @@ +import { Container, type FederatedPointerEvent, Graphics, Text } from 'pixi.js'; +import type { StoredSound } from '../../core/store/SoundStore'; +import { + Button, + drawControlBox, + formatBytes, + HStack, + Label, + Popup, + pressCount, + ScrollContainer, + type Size, + Spacer, + Theme, + textStyle, + truncate, + Widget, +} from '../../ui'; +import { soundLibrary } from '../../viewport/SoundLibrary'; +import { type SoundImportDeps, importStoredSound, pickAndImportSounds } from '../soundImport'; + +/** The dialog keeps this size whatever the list turns out to be, so it does not jump on opening. */ +const LIST_WIDTH = 560; +const LIST_HEIGHT = 340; +const ROW_HEIGHT = 44; +const ROW_GAP = 4; +const ROW_PAD = 8; +/** The play glyph's column at the left of a row: a press in here plays instead of selecting. */ +const PLAY_WIDTH = 34; + +/** One offered sound, with the asset that already uses it (if any). */ +interface PickerItem extends StoredSound { + assetName: string | null; +} + +/** + * The rows. A sound has nothing to show, so each row shows what is known about the file and plays + * it on demand: the ▶ at its left is the preview, the rest of the row is the pick. + */ +class SoundList extends Widget { + private items: PickerItem[] = []; + private rows: { box: Container; background: Graphics; play: Text; caption: Text; detail: Text }[] = []; + private selectedFile: string | null = null; + private readonly empty = new Text({ text: '', style: textStyle('muted') }); + onPick?: (file: string) => void; + onActivate?: (file: string) => void; + emptyText = ''; + /** The last file a ▶ started, so a check can tell a preview from a pick (test hook). */ + lastPlayed: string | null = null; + + constructor() { + super(); + this.addChild(this.empty); + this.on('pointerdown', (e: FederatedPointerEvent) => this.onPress(e)); + } + + setItems(items: PickerItem[]): void { + this.items = items; + for (const row of this.rows) row.box.destroy({ children: true }); + this.rows = items.map(() => { + // A Container holding the background, not a Graphics with children: Pixi v8 only lets + // containers have them (`addChild: Only Containers will be allowed to add children`). + const box = new Container(); + const background = new Graphics(); + const play = new Text({ text: '▶', style: textStyle('accent') }); + const caption = new Text({ style: textStyle('body') }); + const detail = new Text({ style: textStyle('small') }); + box.addChild(background, play, caption, detail); + this.addChild(box); + return { box, background, play, caption, detail }; + }); + if (!items.some((item) => item.file === this.selectedFile)) this.selectedFile = null; + this.layout(); + this.redraw(); + } + + get selected(): PickerItem | null { + return this.items.find((item) => item.file === this.selectedFile) ?? null; + } + + get fileList(): string[] { + return this.items.map((item) => item.file); + } + + get usedFiles(): string[] { + return this.items.filter((item) => item.assetName !== null).map((item) => item.file); + } + + /** The captions as drawn, read off the rows themselves rather than recomputed (test hook). */ + get captions(): string[] { + return this.rows.map((row) => row.caption.text); + } + + select(file: string | null): void { + if (file === this.selectedFile) return; + this.selectedFile = file; + this.redraw(); + if (file) this.onPick?.(file); + } + + /** Global centre of a row, clear of the ▶ column; null when the file is not offered (test hook). */ + rowGlobal(file: string): { x: number; y: number } | null { + const index = this.items.findIndex((item) => item.file === file); + const row = this.rows[index]; + if (!row) return null; + const p = row.box.getGlobalPosition(); + return { x: p.x + PLAY_WIDTH + (this.w - PLAY_WIDTH) / 2, y: p.y + ROW_HEIGHT / 2 }; + } + + /** Global centre of a row's ▶ – what a check presses to hear one without choosing it. */ + playGlobal(file: string): { x: number; y: number } | null { + const index = this.items.findIndex((item) => item.file === file); + const row = this.rows[index]; + if (!row) return null; + const p = row.box.getGlobalPosition(); + return { x: p.x + PLAY_WIDTH / 2, y: p.y + ROW_HEIGHT / 2 }; + } + + override measure(): Size { + return { w: this.minWidth, h: Math.max(this.minHeight, heightFor(this.items.length)) }; + } + + override layout(): void { + this.rows.forEach((row, index) => row.box.position.set(0, index * (ROW_HEIGHT + ROW_GAP))); + } + + override redraw(): void { + this.empty.text = this.items.length === 0 ? this.emptyText : ''; + this.empty.position.set(0, 0); + + this.rows.forEach((node, index) => { + const item = this.items[index]; + if (!item) return; + const selected = item.file === this.selectedFile; + node.background.clear(); + drawControlBox(node.background, this.w, ROW_HEIGHT, { + fill: selected ? Theme.colors.selection : Theme.colors.control, + border: selected ? Theme.colors.accent : Theme.colors.border, + }); + node.play.position.set(ROW_PAD + 2, (ROW_HEIGHT - node.play.height) / 2); + + // The asset name wins where there is one – it is what the project calls this sound. Otherwise + // the name it was uploaded under, which is all a bare hash leaves to go on. + node.caption.style = textStyle(item.assetName ? 'accent' : 'body'); + node.caption.text = truncate( + item.assetName ?? item.name, + node.caption.style, + this.w - PLAY_WIDTH - ROW_PAD, + ); + node.caption.position.set(PLAY_WIDTH, 6); + + node.detail.text = truncate( + item.assetName && item.name ? `${item.name} – ${formatBytes(item.bytes)}` : formatBytes(item.bytes), + node.detail.style, + this.w - PLAY_WIDTH - ROW_PAD, + ); + node.detail.position.set(PLAY_WIDTH, ROW_HEIGHT - 16); + }); + } + + private onPress(e: FederatedPointerEvent): void { + const local = this.toLocal(e.global); + const index = Math.floor(local.y / (ROW_HEIGHT + ROW_GAP)); + const item = local.x >= 0 && local.x <= this.w ? this.items[index] : undefined; + if (!item) return; + if (local.x < PLAY_WIDTH) { + this.lastPlayed = item.file; + soundLibrary.play(item.file, 1, false); + return; + } + this.select(item.file); + if (pressCount(e, item.file) === 2) this.onActivate?.(item.file); + } +} + +function heightFor(count: number): number { + return count === 0 ? 0 : count * ROW_HEIGHT + (count - 1) * ROW_GAP; +} + +/** + * Offers every sound the server holds, the way the font picker offers typefaces: uploading the same + * file twice is wasteful, and nothing collects the files a project stopped referencing. + */ +export class SoundPicker extends Popup { + private readonly list = new SoundList(); + private readonly scroll = new ScrollContainer(); + private readonly status: Label; + private readonly detail: Label; + private readonly use: Button; + + constructor(private readonly deps: SoundImportDeps) { + super({ modal: true, padding: Theme.padding * 2, gap: Theme.gap }); + this.content.add(new Label('Choose a sound', { style: 'bodyBold' })); + this.status = new Label('Loading sounds…', { style: 'muted' }); + this.content.add(this.status); + + this.list.minWidth = LIST_WIDTH; + this.list.emptyText = 'No sounds on the server yet – upload one.'; + this.list.onPick = () => this.refreshSelection(); + this.list.onActivate = () => this.confirm(); + this.scroll.minWidth = LIST_WIDTH; + this.scroll.minHeight = LIST_HEIGHT; + this.scroll.setContent(this.list); + this.content.add(this.scroll); + + this.detail = new Label('Press ▶ to hear one.', { style: 'muted' }); + this.content.add(this.detail); + + const buttons = new HStack({ gap: Theme.gap, align: 'center' }); + const upload = new Button('Upload…'); + upload.onClick = this.closeThen(() => pickAndImportSounds(this.deps)); + buttons.add(upload); + buttons.add(new Spacer(), 1); + const cancel = new Button('Cancel'); + cancel.onClick = this.closeThen(); + buttons.add(cancel); + this.use = new Button('Use sound', { variant: 'primary' }); + this.use.onClick = () => this.confirm(); + this.use.setEnabled(false); + buttons.add(this.use); + this.content.add(buttons); + + void this.loadList(); + } + + /** What the dialog offers, which of it the project already uses, and what is picked (test hook). */ + debugInfo(): { + files: string[]; + used: string[]; + captions: string[]; + selected: string | null; + played: string | null; + } { + return { + files: this.list.fileList, + used: this.list.usedFiles, + captions: this.list.captions, + selected: this.list.selected?.file ?? null, + played: this.list.lastPlayed, + }; + } + + rowGlobal(file: string): { x: number; y: number } | null { + return this.list.rowGlobal(file); + } + + playGlobal(file: string): { x: number; y: number } | null { + return this.list.playGlobal(file); + } + + override close(): void { + super.close(); + if (current === this) current = null; + // A preview left running would outlive the dialog that started it, with nothing left to stop it. + soundLibrary.stopAll(); + } + + private async loadList(): Promise { + try { + const stored = await this.deps.sounds.list(); + const named = new Map(); + for (const asset of this.deps.state.project.assets) { + if (asset.type === 'sound') named.set(asset.file, asset.name); + } + this.list.setItems(stored.map((item) => ({ ...item, assetName: named.get(item.file) ?? null }))); + this.scroll.minHeight = Math.min(LIST_HEIGHT, Math.max(ROW_HEIGHT, heightFor(stored.length))); + this.status.setText( + stored.length === 1 ? '1 sound on the server' : `${stored.length} sounds on the server`, + ); + } catch (error) { + this.status.setText( + `Could not list the sounds: ${error instanceof Error ? error.message : String(error)}`, + ); + } + this.refreshSelection(); + } + + private refreshSelection(): void { + const item = this.list.selected; + this.use.setEnabled(item !== null); + this.detail.setText(item ? describe(item) : 'Press ▶ to hear one.'); + this.refreshLayout(); + } + + private confirm(): void { + const item = this.list.selected; + if (!item) return; + this.close(); + importStoredSound(item, this.deps); + } +} + +function describe(item: PickerItem): string { + const size = item.name ? `${item.name} – ${formatBytes(item.bytes)}` : formatBytes(item.bytes); + return item.assetName ? `${size} – already in the project as "${item.assetName}"` : size; +} + +let current: SoundPicker | null = null; + +export function openSoundPicker(deps: SoundImportDeps): SoundPicker { + current?.close(); + current = new SoundPicker(deps); + current.openCentered(); + return current; +} + +/** The open dialog, for the test hook. */ +export function currentSoundPicker(): SoundPicker | null { + return current; +} diff --git a/src/editor/sound/SoundPreview.ts b/src/editor/sound/SoundPreview.ts new file mode 100644 index 00000000..c99ee1e4 --- /dev/null +++ b/src/editor/sound/SoundPreview.ts @@ -0,0 +1,44 @@ +import { Button, HStack, Label, Theme } from '../../ui'; +import { soundLibrary } from '../../viewport/SoundLibrary'; + +/** + * What a sound asset looks like: a button that plays it and the length the decoder reports. There + * is nothing else to show – its file name is a hash, and the waveform would be a picture of bytes + * nobody reads. + * + * Playing here goes through the same library a running game uses, so what the inspector plays is + * exactly what the graph will: same decode, same device, and `Stop` reaches both. + */ +export class SoundPreview extends HStack { + private readonly length: Label; + + constructor(private readonly file: string) { + super({ gap: Theme.gap, align: 'center' }); + const play = new Button('▶ Play'); + play.onClick = () => soundLibrary.play(this.file, 1, false); + this.add(play); + const stop = new Button('■ Stop'); + stop.onClick = () => soundLibrary.stop(this.file); + this.add(stop); + this.length = new Label('', { style: 'muted' }); + this.add(this.length, 1); + this.showLength(); + } + + /** + * The length is only known once the bytes are decoded, and decoding needs an audio device. So it + * is asked for rather than waited on: a preview nobody has played yet says nothing about length, + * which is more honest than a number guessed from the file size. + */ + private showLength(): void { + const known = soundLibrary.duration(this.file); + if (known !== null) { + this.length.setText(`${known.toFixed(2)} s`); + return; + } + void soundLibrary.load(this.file).then((buffer) => { + if (this.destroyed) return; + this.length.setText(buffer ? `${buffer.duration.toFixed(2)} s` : 'Could not be decoded.'); + }); + } +} diff --git a/src/editor/soundImport.ts b/src/editor/soundImport.ts new file mode 100644 index 00000000..0eb58c56 --- /dev/null +++ b/src/editor/soundImport.ts @@ -0,0 +1,105 @@ +import { CreateSoundCommand } from '../core/commands/assetCommands'; +import { nextAssetName } from '../core/commands/naming'; +import type { SoundAsset } from '../core/model/Asset'; +import type { SoundStore, UploadedSound } from '../core/store/SoundStore'; +import type { EditorState } from './state'; +import { type ImportOutcome, assetNameForFile, importNotice } from './textureImport'; + +/** What the file dialog offers and a drop accepts; the server rejects everything else anyway. */ +export const SOUND_ACCEPT = '.mp3,.ogg,.wav,.m4a'; + +const SOUND_EXTENSION = /\.(mp3|ogg|wav|m4a)$/i; + +/** + * Sounds are told apart by their name, not their MIME type: browsers report a dropped `.m4a` as + * `audio/mp4`, `audio/x-m4a` or nothing at all, and an `.ogg` as `audio/ogg` or `video/ogg`. + */ +export function isSoundFileName(name: string): boolean { + return SOUND_EXTENSION.test(name); +} + +export interface SoundImportDeps { + state: EditorState; + sounds: SoundStore; +} + +/** Sequential like `importImages`, so each asset is named against the project as it then stands. */ +export async function importSounds(files: readonly File[], deps: SoundImportDeps): Promise { + if (files.length === 0) return []; + const outcomes: ImportOutcome[] = []; + for (const file of files) outcomes.push(await importSound(file, deps)); + const created = outcomes.filter((outcome) => outcome.assetId !== null); + const last = created[created.length - 1]; + if (last?.assetId) deps.state.select({ kind: 'asset', assetId: last.assetId }); + deps.state.setNotice(importNotice(outcomes, 'sound')); + return outcomes; +} + +async function importSound(file: File, deps: SoundImportDeps): Promise { + try { + if (!isSoundFileName(file.name)) throw new Error('Not a sound file – use MP3, OGG, WAV or M4A.'); + // The bytes go up untouched: nothing here could encode audio, and a re-encode would be a loss. + const uploaded = await deps.sounds.upload(file, file.type || 'application/octet-stream', file.name); + return adopt(uploaded, assetNameForFile(file.name, 'Sound'), file.name, deps); + } catch (error) { + return { + name: file.name, + assetId: null, + error: error instanceof Error ? error.message : String(error), + }; + } +} + +/** Makes an asset for a sound the server holds, or reuses the one already naming those bytes. */ +function adopt(sound: UploadedSound, nameBase: string, label: string, deps: SoundImportDeps): ImportOutcome { + const existing = deps.state.project.assets.find( + (asset): asset is SoundAsset => asset.type === 'sound' && asset.file === sound.file, + ); + if (existing) return { name: label, assetId: existing.id, error: null, reused: true }; + + const name = nextAssetName(deps.state.project, nameBase); + const command = new CreateSoundCommand(name, sound.file); + // A rejected command has already put its reason into the notice; that is the message here. + if (!deps.state.execute(command)) return { name: label, assetId: null, error: deps.state.notice }; + return { name: label, assetId: command.asset.id, error: null, reused: false }; +} + +/** + * Adopts a sound the server already holds – no upload, the bytes are there. Chosen in the picker, + * so this is "+ → Sound" and the asset is selected the way a created one would be. + */ +export function importStoredSound(sound: UploadedSound, deps: SoundImportDeps): ImportOutcome { + const outcome = adopt(sound, assetNameForFile(sound.name, 'Sound'), sound.name || sound.file, deps); + if (outcome.assetId) deps.state.select({ kind: 'asset', assetId: outcome.assetId }); + deps.state.setNotice( + outcome.error ?? + (outcome.reused ? 'That sound is already in the project.' : 'Added the sound to the project.'), + ); + return outcome; +} + +/** Opens the browser's file dialog; resolves empty when the dialog was cancelled. */ +export function pickSoundFiles(): Promise { + return new Promise((resolve) => { + const input = document.createElement('input'); + input.type = 'file'; + input.accept = SOUND_ACCEPT; + input.multiple = true; + input.style.display = 'none'; + document.body.appendChild(input); + const finish = (files: File[]) => { + input.remove(); + resolve(files); + }; + input.addEventListener('change', () => finish([...(input.files ?? [])])); + // Without `cancel` a dismissed dialog leaves both the element and this promise behind. + input.addEventListener('cancel', () => finish([])); + input.click(); + }); +} + +export function pickAndImportSounds(deps: SoundImportDeps): void { + void pickSoundFiles() + .then((files) => importSounds(files, deps)) + .catch((error: unknown) => deps.state.setNotice(String(error), 'error')); +} diff --git a/src/runtime/systems/Audio.ts b/src/runtime/systems/Audio.ts new file mode 100644 index 00000000..44f62ec0 --- /dev/null +++ b/src/runtime/systems/Audio.ts @@ -0,0 +1,19 @@ +/** + * What the runtime needs of the outside world to make a noise – the counterpart of `Input` on the + * way out. The browser side of it is `viewport/SoundLibrary`; a test (and a run started before + * anything wired one in) gets `SILENT_AUDIO` and plays to nobody. + * + * Files, not asset ids: resolving the id is the `BehaviorSystem`'s job, because it is the one + * holding the project. Everything below this line only knows bytes and voices. + */ +export interface AudioSink { + /** Starts one voice. `volume` is final (0–1) and every call overlaps whatever is already playing. */ + play(file: string, volume: number, loop: boolean): void; + /** Silences every voice of that file; `''` silences all of them. */ + stop(file: string): void; +} + +export const SILENT_AUDIO: AudioSink = { + play: () => {}, + stop: () => {}, +}; diff --git a/src/runtime/systems/BehaviorSystem.ts b/src/runtime/systems/BehaviorSystem.ts index 28cfa4a2..0f8f036b 100644 --- a/src/runtime/systems/BehaviorSystem.ts +++ b/src/runtime/systems/BehaviorSystem.ts @@ -19,7 +19,11 @@ import { stepKeyRepeat } from '../../core/graph/keyRepeat'; import { graphLookup } from '../../core/model/graphLookup'; import { type PropertyBag, textOf } from '../../core/model/Properties'; import { findAssetOf, type Project } from '../../core/model/Project'; +import { soundFileOf } from '../../core/model/sound'; import { parentChain } from '../../core/model/walk'; +import { resolveAssetProps } from '../../core/properties/resolve'; +import { loopsOf, volumeOf } from '../../core/properties/sounds'; +import { type AudioSink, SILENT_AUDIO } from './Audio'; import type { Entity } from '../Entity'; import type { World } from '../World'; import { bodyOf, type CollisionPair, CollisionSystem } from './CollisionSystem'; @@ -80,10 +84,11 @@ export function addGameSystems( project: Project, input: Input, shouldRun?: (entity: Entity) => boolean, + audio: AudioSink = SILENT_AUDIO, ): BehaviorSystem { const collisions = new CollisionSystem(); const animations = new AnimationSystem(); - const behaviors = new BehaviorSystem(project, input, { collisions, animations }, shouldRun); + const behaviors = new BehaviorSystem(project, input, { collisions, animations, audio }, shouldRun); world.addSystem(collisions); world.addSystem(animations); world.addSystem(behaviors); @@ -94,6 +99,8 @@ export function addGameSystems( export interface EventSources { collisions?: CollisionSystem | null; animations?: AnimationSystem | null; + /** Where `Play sound` ends up. Absent is a run that plays silently, which is what a test wants. */ + audio?: AudioSink | null; } /** The behavior failure that stopped a run, if that is why it stopped. */ @@ -111,6 +118,7 @@ export class BehaviorSystem implements System { private declared: Record = {}; private readonly collisions: CollisionSystem | null; private readonly animations: AnimationSystem | null; + private readonly audio: AudioSink; constructor( private readonly project: Project, @@ -124,6 +132,7 @@ export class BehaviorSystem implements System { ) { this.collisions = sources.collisions ?? null; this.animations = sources.animations ?? null; + this.audio = sources.audio ?? SILENT_AUDIO; } /** The values `global` declarations share this run – read-only to callers, rebuilt by `start`. */ @@ -440,6 +449,19 @@ export class BehaviorSystem implements System { return merged; } + /** + * What the graph asked for, as the sink understands it: the asset's own volume scaled by the + * node's, and its loop flag unless the node overrules it either way. The asset is what a sound + * *is* – a node that says nothing therefore plays it as the project describes it. + */ + private playSound(assetId: string, volume: number, loop: boolean | undefined): void { + const asset = findAssetOf(this.project, assetId, 'sound'); + if (!asset?.file) return; + const props = resolveAssetProps(this.project, asset); + const level = Math.max(0, Math.min(1, volumeOf(props) * volume)); + this.audio.play(asset.file, level, loop ?? loopsOf(props)); + } + /** Runs graph code for one instance; an exception stops play and is logged, never thrown. */ private run( instance: Instance, @@ -499,6 +521,13 @@ export class BehaviorSystem implements System { // eslint-disable-next-line no-console console.log(`[behavior ${entity.name}]`, message); }, + playSound: (assetId, volume, loop) => this.playSound(assetId, volume, loop), + // An id nothing resolves stops nothing, rather than silencing the whole run by accident; + // only the empty id means "everything", which is what the node promises. + stopSound: (assetId) => { + const file = assetId === '' ? '' : soundFileOf(this.project, assetId); + if (assetId === '' || file) this.audio.stop(file); + }, spawn: (assetId, position, parent) => world.spawn(assetId, position, resolve(parent) ?? null), destroy: (target) => { const found = resolve(target); diff --git a/src/viewport/SoundLibrary.ts b/src/viewport/SoundLibrary.ts new file mode 100644 index 00000000..7af11405 --- /dev/null +++ b/src/viewport/SoundLibrary.ts @@ -0,0 +1,171 @@ +import type { AudioSink } from '../runtime/systems/Audio'; + +const DEFAULT_BASE = '/game-maker/data/sounds'; + +/** One sound in flight: its source, its own gain, and the file it came from. */ +interface Voice { + file: string; + source: AudioBufferSourceNode; + gain: GainNode; +} + +/** + * Plays uploaded sounds, keyed by file name – the texture library's twin on the audio side. Files + * are content addressed, so a decoded buffer can never go stale and nothing is evicted. + * + * Everything here is asynchronous on the way in and synchronous on the way out: `play` returns at + * once and starts the voice when the bytes have been fetched and decoded. A sound fired in the + * first frame of a run is therefore heard a moment later rather than not at all, and a graph never + * waits on the network. + * + * The context is created on the first play, not at start-up: a browser refuses one before the page + * has been interacted with, and an editor that was only ever looked at should not be holding an + * audio device open. + */ +export class SoundLibrary implements AudioSink { + private base = DEFAULT_BASE; + private context: AudioContext | null = null; + private readonly buffers = new Map(); + private readonly loading = new Map>(); + /** Never retried, so one broken reference cannot spam the network. */ + private readonly failed = new Set(); + private voices: Voice[] = []; + private started = 0; + private stopped = 0; + /** The last voice actually started, with the values it was started at – what a check reads back. */ + private last: { file: string; volume: number; loop: boolean } | null = null; + /** + * How often a stop has reached each file, and how often one has silenced everything. Counters + * rather than a "cancelled" flag: what a play has to know is whether a stop happened *after it + * was asked for*, and a flag cannot tell that from one that happened long before – which would + * make the first play of a sound disappear whenever anything had ever stopped it. + */ + private readonly stops = new Map(); + private allStops = 0; + + /** Where the files are served from. Set once at start-up, before anything plays. */ + configure(base: string): void { + this.base = base.replace(/\/+$/, ''); + } + + play(file: string, volume: number, loop: boolean): void { + if (file === '' || this.failed.has(file)) return; + const buffer = this.buffers.get(file); + if (buffer) { + this.start(file, buffer, volume, loop); + return; + } + const askedAt = this.stopsFor(file); + void this.load(file).then((loaded) => { + // A `Stop sound` that happened while the bytes were in flight has to win, or a sound the + // graph already silenced would start afterwards and play on its own. + if (loaded && this.stopsFor(file) === askedAt) this.start(file, loaded, volume, loop); + }); + } + + stop(file: string): void { + const going = file === '' ? this.voices : this.voices.filter((voice) => voice.file === file); + for (const voice of going) voice.source.stop(); + // `onended` removes each one, but not before the next frame – the count has to be right now. + this.voices = this.voices.filter((voice) => !going.includes(voice)); + this.stopped += going.length; + if (file === '') this.allStops += 1; + else this.stops.set(file, (this.stops.get(file) ?? 0) + 1); + } + + /** Every voice at once: what ending a run does, so nothing outlives the world that started it. */ + stopAll(): void { + this.stop(''); + } + + /** A pause holds the sounds where they are; resuming carries them on mid-note. */ + setSuspended(suspended: boolean): void { + const context = this.context; + if (!context) return; + if (suspended) void context.suspend(); + else void context.resume(); + } + + /** What has been played and what is playing – the only witness a muted check script has. */ + stats(): { + playing: number; + started: number; + stopped: number; + decoded: number; + failed: number; + last: { file: string; volume: number; loop: boolean } | null; + } { + return { + playing: this.voices.length, + started: this.started, + stopped: this.stopped, + decoded: this.buffers.size, + failed: this.failed.size, + last: this.last, + }; + } + + /** How long a decoded sound runs, in seconds; null while it has not been loaded (test hook). */ + duration(file: string): number | null { + return this.buffers.get(file)?.duration ?? null; + } + + /** Fetches and decodes once per file; a second play while it is in flight waits on the first. */ + async load(file: string): Promise { + const held = this.buffers.get(file); + if (held) return held; + if (this.failed.has(file)) return null; + const running = this.loading.get(file); + if (running) return running; + const work = (async () => { + try { + const response = await fetch(`${this.base}/${file}`); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const bytes = await response.arrayBuffer(); + const buffer = await this.audio().decodeAudioData(bytes); + this.buffers.set(file, buffer); + return buffer; + } catch (error: unknown) { + this.failed.add(file); + console.error(`[sounds] cannot load ${file}`, error); + return null; + } finally { + this.loading.delete(file); + } + })(); + this.loading.set(file, work); + return work; + } + + private start(file: string, buffer: AudioBuffer, volume: number, loop: boolean): void { + const context = this.audio(); + // A context starts suspended where the page has not been interacted with; without this the + // voice would run silently and end as if it had played. + if (context.state === 'suspended') void context.resume(); + const source = context.createBufferSource(); + source.buffer = buffer; + source.loop = loop; + const gain = context.createGain(); + gain.gain.value = Math.max(0, Math.min(1, volume)); + this.last = { file, volume: gain.gain.value, loop }; + source.connect(gain).connect(context.destination); + const voice: Voice = { file, source, gain }; + source.onended = () => { + this.voices = this.voices.filter((other) => other !== voice); + }; + this.voices.push(voice); + this.started += 1; + source.start(); + } + + private audio(): AudioContext { + return (this.context ??= new AudioContext()); + } + + /** Stops this file has seen, its share of the "everything" ones included. */ + private stopsFor(file: string): number { + return (this.stops.get(file) ?? 0) + this.allStops; + } +} + +export const soundLibrary = new SoundLibrary(); diff --git a/tests/core/node-catalog.test.ts b/tests/core/node-catalog.test.ts index cc7cb9d9..90f35faf 100644 --- a/tests/core/node-catalog.test.ts +++ b/tests/core/node-catalog.test.ts @@ -415,6 +415,10 @@ const CASES: Record = { { name: 'outputs the id it was set to', params: { value: 'sm-1' }, outputs: { value: 'sm-1' } }, { name: 'unset is no machine', outputs: { value: '' } }, ], + Sound: [ + { name: 'outputs the id it was set to', params: { value: 'snd-1' }, outputs: { value: 'snd-1' } }, + { name: 'unset is no sound', outputs: { value: '' } }, + ], Add: [ { name: 'adds', inputs: { a: 2, b: 3 }, outputs: { result: 5 } }, { name: 'numeric text is coerced', inputs: { a: '2', b: 3 }, outputs: { result: 5 } }, @@ -794,6 +798,49 @@ const CASES: Record = { after: () => expect(emitTarget.emits).toBe(1), }, ], + PlaySound: [ + { + name: 'no sound is a no-op that still continues the flow', + inputs: { sound: '', volume: 1, loop: '' }, + next: 'exec', + after: (ctx) => expect(ctx.played).toEqual([]), + }, + { + name: 'hands the id on with the volume and loop it was given', + inputs: { sound: 'snd-1', volume: 0.5, loop: 'true' }, + next: 'exec', + // What the two mean – a factor on the asset's volume, an answer that overrules its loop + // flag – is the BehaviorSystem's to resolve, since only it holds the project (see + // `sounds.test.ts`). + after: (ctx) => expect(ctx.played).toEqual([{ assetId: 'snd-1', volume: 0.5, loop: true }]), + }, + { + name: 'saying nothing about looping leaves the answer to the asset', + inputs: { sound: 'snd-1', volume: 1, loop: '' }, + next: 'exec', + after: (ctx) => expect(ctx.played[0]?.loop).toBeUndefined(), + }, + { + name: 'off is an answer of its own, not the absence of one', + inputs: { sound: 'snd-1', volume: 1, loop: 'false' }, + next: 'exec', + after: (ctx) => expect(ctx.played[0]?.loop).toBe(false), + }, + ], + StopSound: [ + { + name: 'silences the sound it names', + inputs: { sound: 'snd-1' }, + next: 'exec', + after: (ctx) => expect(ctx.silenced).toEqual(['snd-1']), + }, + { + name: 'naming none asks for everything', + inputs: { sound: '' }, + next: 'exec', + after: (ctx) => expect(ctx.silenced).toEqual(['']), + }, + ], Log: [ { name: 'logs its message and continues', diff --git a/tests/core/serialize.test.ts b/tests/core/serialize.test.ts index f6de9b72..b45b1a94 100644 --- a/tests/core/serialize.test.ts +++ b/tests/core/serialize.test.ts @@ -77,8 +77,9 @@ describe('serialize', () => { ], }, { id: 'sp1', type: 'sprite', props: { width: 10 } }, - // An asset type this build does not know (a newer document, a renamed type). - { id: 'x', type: 'sound' }, + // An asset type this build does not know (a newer document, a renamed type). It used to + // be `sound`, which this build does know since #45 – hence a type nothing has claimed. + { id: 'x', type: 'video' }, { type: 'camera' }, ], }, diff --git a/tests/core/sounds.test.ts b/tests/core/sounds.test.ts new file mode 100644 index 00000000..4b499d54 --- /dev/null +++ b/tests/core/sounds.test.ts @@ -0,0 +1,231 @@ +/** + * A sound is an asset holding an uploaded file. Nothing wears it and nothing places it – a graph + * plays it, and what the asset says about volume and looping is what a bare `Play sound` does. + */ +import { describe, expect, it } from 'vitest'; +import { + buildAsset, + CreateAssetCommand, + CreateSoundCommand, + SetAssetPropertyCommand, +} from '../../src/core/commands/assetCommands'; +import type { BehaviorAsset, SoundAsset } from '../../src/core/model/Asset'; +import { isPlaceableType } from '../../src/core/model/AssetTypes'; +import { isSoundFile, soundFileOf } from '../../src/core/model/sound'; +import { assetRefChoices } from '../../src/core/properties/assetRef'; +import { schemaFor } from '../../src/core/properties/PropertySchema'; +import { LOOP_KEY, loopsOf, VOLUME_KEY, volumeOf } from '../../src/core/properties/sounds'; +import { deserializeProject } from '../../src/core/store/serialize'; +import type { AudioSink } from '../../src/runtime/systems/Audio'; +import { + graphEditor, + makeBehavior, + makeProject, + makeSpriteScene, + place, + roundTrip, + type SceneFixture, + startWorld, +} from '../helpers'; + +const FILE = `${'a1'.repeat(32)}.wav`; + +/** Records what it was asked to play instead of making a noise – the AudioSink a test wants. */ +function fakeAudio(): AudioSink & { + played: { file: string; volume: number; loop: boolean }[]; + stopped: string[]; +} { + const sink = { + played: [] as { file: string; volume: number; loop: boolean }[], + stopped: [] as string[], + play: (file: string, volume: number, loop: boolean) => sink.played.push({ file, volume, loop }), + stop: (file: string) => sink.stopped.push(file), + }; + return sink; +} + +function makeSound(name = 'Jump', file = FILE) { + const fixture = makeProject(); + const command = new CreateSoundCommand(name, file); + fixture.stack.execute(command); + return { ...fixture, sound: command.asset }; +} + +/** A scene whose one behavior plays `sound` on start, with whatever the node itself carries. */ +function playingScene(params: Record = {}) { + const f: SceneFixture & { behavior?: BehaviorAsset } = makeSpriteScene('Box'); + const create = new CreateSoundCommand('Jump', FILE); + f.stack.execute(create); + const behavior = makeBehavior(f, 'Logic'); + const box = place(f, f.spriteId, { x: 0, y: 0 }); + place(f, behavior.id, { x: 0, y: 0 }, box); + const edit = graphEditor(f, behavior); + const start = edit.addNode('OnStart'); + const play = edit.addNode('PlaySound', { sound: create.asset.id, volume: 1, ...params }); + edit.connect(start, 'exec', play, 'exec'); + return { ...f, sound: create.asset, behavior, play }; +} + +describe('sound files', () => { + it('accepts only a content-addressed name in a supported format', () => { + expect(isSoundFile(FILE)).toBe(true); + expect(isSoundFile(`${'b3'.repeat(32)}.mp3`)).toBe(true); + expect(isSoundFile(`${'b3'.repeat(32)}.m4a`)).toBe(true); + expect(isSoundFile(`${'a1'.repeat(32)}.png`)).toBe(false); + expect(isSoundFile(`${'a1'.repeat(32)}.WAV`)).toBe(false); + expect(isSoundFile('jump.wav')).toBe(false); + expect(isSoundFile('')).toBe(false); + }); +}); + +describe('the sound asset', () => { + it('is not placeable, starts without a file, and is full volume and one-shot by default', () => { + expect(isPlaceableType('sound')).toBe(false); + const asset = buildAsset('sound', 'Sound'); + expect(asset).toMatchObject({ type: 'sound', file: '' }); + expect(volumeOf(asset.props)).toBe(1); + expect(loopsOf(asset.props)).toBe(false); + expect(schemaFor('sound').map((def) => def.key)).toEqual([VOLUME_KEY, LOOP_KEY]); + }); + + it('refuses a file name the server could not have produced', () => { + const fixture = makeProject(); + expect(() => fixture.stack.execute(new CreateSoundCommand('Bad', 'jump.wav'))).toThrow( + 'is not a stored sound file', + ); + expect(fixture.project.assets).toHaveLength(0); + }); + + it('survives a round trip and drops a file name it cannot trust', () => { + const s = makeSound(); + expect(roundTrip(s.project)).toEqual(s.project); + + const repaired = deserializeProject({ + name: 'P', + assets: [ + { id: 's1', type: 'sound', name: 'Local', file: '../../etc/passwd' }, + { id: 's2', type: 'sound', name: 'Gone' }, + ], + }); + expect(repaired.assets.map((a) => (a as SoundAsset).file)).toEqual(['', '']); + }); + + it('resolves to a file, and to none when the id is unset, dangling or of another type', () => { + const s = makeSound(); + expect(soundFileOf(s.project, s.sound.id)).toBe(FILE); + expect(soundFileOf(s.project, '')).toBe(''); + expect(soundFileOf(s.project, 'no-such-asset')).toBe(''); + + const text = new CreateAssetCommand('text', 'Title'); + s.stack.execute(text); + expect(soundFileOf(s.project, text.asset.id)).toBe(''); + }); + + it('is what a sound picker offers, and nothing else is', () => { + const s = makeSound(); + s.stack.execute(new CreateAssetCommand('text', 'Title')); + expect(assetRefChoices(s.project, ['sound']).map((a) => a.id)).toEqual([s.sound.id]); + }); +}); + +describe('Play sound', () => { + it('plays the asset as the project describes it', () => { + const f = playingScene(); + const audio = fakeAudio(); + startWorld(f, { behaviors: true, audio }).frame(); + expect(audio.played).toEqual([{ file: FILE, volume: 1, loop: false }]); + }); + + it('scales the asset volume rather than replacing it, and clamps the product', () => { + const f = playingScene({ volume: 0.5 }); + f.stack.execute(new SetAssetPropertyCommand(f.sound.id, VOLUME_KEY, 0.4)); + const audio = fakeAudio(); + startWorld(f, { behaviors: true, audio }).frame(); + expect(audio.played[0]?.volume).toBeCloseTo(0.2); + + const loud = playingScene({ volume: 10 }); + const loudAudio = fakeAudio(); + startWorld(loud, { behaviors: true, audio: loudAudio }).frame(); + expect(loudAudio.played[0]?.volume).toBe(1); + }); + + it('loops as the asset says while the node says nothing', () => { + const asset = playingScene(); + asset.stack.execute(new SetAssetPropertyCommand(asset.sound.id, LOOP_KEY, true)); + const assetAudio = fakeAudio(); + startWorld(asset, { behaviors: true, audio: assetAudio }).frame(); + expect(assetAudio.played[0]?.loop).toBe(true); + + const node = playingScene({ loop: 'true' }); + const nodeAudio = fakeAudio(); + startWorld(node, { behaviors: true, audio: nodeAudio }).frame(); + expect(nodeAudio.played[0]?.loop).toBe(true); + }); + + /** The reason the node asks for three answers rather than a checkbox: music played as a one-shot. */ + it('plays a looping asset once when the node says off', () => { + const f = playingScene({ loop: 'false' }); + f.stack.execute(new SetAssetPropertyCommand(f.sound.id, LOOP_KEY, true)); + const audio = fakeAudio(); + startWorld(f, { behaviors: true, audio }).frame(); + expect(audio.played[0]?.loop).toBe(false); + }); + + it('takes a boolean arriving over an edge literally', () => { + const on = playingScene(); + const onEdit = graphEditor(on, on.behavior); + onEdit.connect(onEdit.addNode('Boolean', { value: true }), 'value', on.play, 'loop'); + const onAudio = fakeAudio(); + startWorld(on, { behaviors: true, audio: onAudio }).frame(); + expect(onAudio.played[0]?.loop).toBe(true); + + const off = playingScene(); + off.stack.execute(new SetAssetPropertyCommand(off.sound.id, LOOP_KEY, true)); + const offEdit = graphEditor(off, off.behavior); + offEdit.connect(offEdit.addNode('Boolean', { value: false }), 'value', off.play, 'loop'); + const offAudio = fakeAudio(); + startWorld(off, { behaviors: true, audio: offAudio }).frame(); + expect(offAudio.played[0]?.loop).toBe(false); + }); + + it('plays nothing for an id that resolves to no sound, and does not stop the flow', () => { + const f = playingScene(); + const edit = graphEditor(f, f.behavior); + const log = edit.addNode('Log', { message: 'after' }); + edit.connect(f.play, 'exec', log, 'exec'); + f.play.params.sound = 'no-such-asset'; + + const audio = fakeAudio(); + const { world, frame } = startWorld(f, { behaviors: true, audio }); + frame(); + expect(audio.played).toEqual([]); + expect(world.log).toEqual(['after']); + }); +}); + +describe('Stop sound', () => { + it('silences one sound by its file, and everything when it names none', () => { + const f = playingScene(); + const edit = graphEditor(f, f.behavior); + const stop = edit.addNode('StopSound', { sound: f.sound.id }); + const all = edit.addNode('StopSound', { sound: '' }); + edit.connect(f.play, 'exec', stop, 'exec'); + edit.connect(stop, 'exec', all, 'exec'); + + const audio = fakeAudio(); + startWorld(f, { behaviors: true, audio }).frame(); + expect(audio.stopped).toEqual([FILE, '']); + }); + + it('silences nothing at all for an id that resolves to no sound', () => { + const f = playingScene(); + const edit = graphEditor(f, f.behavior); + const stop = edit.addNode('StopSound', { sound: 'no-such-asset' }); + edit.connect(f.play, 'exec', stop, 'exec'); + + const audio = fakeAudio(); + startWorld(f, { behaviors: true, audio }).frame(); + // Not `['']`: a dangling reference must not turn into "stop everything". + expect(audio.stopped).toEqual([]); + }); +}); diff --git a/tests/helpers.ts b/tests/helpers.ts index 11c3711f..08e908bf 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -29,6 +29,7 @@ import type { Position, PropertyBag, Vec2 } from '../src/core/model/Properties'; import { createEmptyProject, findScene, type Project } from '../src/core/model/Project'; import type { SceneObject } from '../src/core/model/SceneObject'; import { deserializeProject, serializeProject } from '../src/core/store/serialize'; +import type { AudioSink } from '../src/runtime/systems/Audio'; import { BehaviorSystem } from '../src/runtime/systems/BehaviorSystem'; import { CollisionSystem } from '../src/runtime/systems/CollisionSystem'; import { AnimationSystem } from '../src/runtime/systems/AnimationSystem'; @@ -193,6 +194,9 @@ export type FakeContext = NodeContext & { logs: string[]; stopped: boolean; changes: { machineId: string; from: string; to: string }[]; + /** What the context was asked to play and to silence, in order – nothing here makes a noise. */ + played: { assetId: string; volume: number; loop: boolean | undefined }[]; + silenced: string[]; }; /** A NodeContext double: self/owner handles, `speed = 100`, dt 0.5, moves applied directly, `log` and `stop` recorded. */ @@ -221,6 +225,10 @@ export function fakeContext(overrides: Partial = {}): FakeContext { return { blockedX: false, blockedY: false }; }, log: (message) => logs.push(message), + played: [], + silenced: [], + playSound: (assetId, volume, loop) => ctx.played.push({ assetId, volume, loop }), + stopSound: (assetId) => ctx.silenced.push(assetId), spawn: () => null, destroy: () => {}, stop: () => { @@ -259,6 +267,8 @@ export interface WorldOptions { collisions?: boolean; animations?: boolean; behaviors?: boolean; + /** Where `Play sound` ends up; without one the run plays to nobody (see `SILENT_AUDIO`). */ + audio?: AudioSink; } export interface StartedWorld { @@ -285,7 +295,7 @@ export function startWorld(fixture: SceneFixture, options: WorldOptions = {}): S if (collisions) world.addSystem(collisions); if (animations) world.addSystem(animations); const behaviors = options.behaviors - ? new BehaviorSystem(fixture.project, input, { collisions, animations }) + ? new BehaviorSystem(fixture.project, input, { collisions, animations, audio: options.audio }) : null; if (behaviors) world.addSystem(behaviors); return { diff --git a/tests/viewport/sound-library.test.ts b/tests/viewport/sound-library.test.ts new file mode 100644 index 00000000..5eb2d98a --- /dev/null +++ b/tests/viewport/sound-library.test.ts @@ -0,0 +1,160 @@ +/** + * The library between `Play sound` and the audio device. Everything here is about the gap between + * asking for a sound and being able to play it: the bytes are fetched and decoded on first use, so + * a stop can land while a play is still in flight. + * + * The device is stubbed – nothing makes a noise in a unit test, and Web Audio does not exist here. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { SoundLibrary } from '../../src/viewport/SoundLibrary'; + +const FILE = 'a.wav'; +const OTHER = 'b.wav'; + +interface FakeSource { + buffer: unknown; + loop: boolean; + onended: (() => void) | null; + started: boolean; + stopped: boolean; + connect: (target: unknown) => unknown; + start: () => void; + stop: () => void; +} + +const sources: FakeSource[] = []; + +/** Resolves the pending decodes by hand, so a test decides when the bytes arrive. */ +let decodes: (() => void)[] = []; + +function fakeContext() { + return { + state: 'running', + destination: {}, + resume: () => Promise.resolve(), + suspend: () => Promise.resolve(), + decodeAudioData: () => new Promise((resolve) => decodes.push(() => resolve({ duration: 1.5 }))), + createBufferSource: () => { + const source: FakeSource = { + buffer: null, + loop: false, + onended: null, + started: false, + stopped: false, + connect: (target: unknown) => target, + start: () => { + source.started = true; + }, + stop: () => { + source.stopped = true; + }, + }; + sources.push(source); + return source; + }, + createGain: () => ({ gain: { value: 1 }, connect: (target: unknown) => target }), + }; +} + +/** Lets the promise chain inside `play` run to the point where it would start a voice. */ +const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); + +beforeEach(() => { + sources.length = 0; + decodes = []; + vi.stubGlobal( + 'AudioContext', + class { + constructor() { + return fakeContext(); + } + }, + ); + vi.stubGlobal('fetch', () => + Promise.resolve({ ok: true, arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)) }), + ); +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +/** Plays `file` and lets its bytes arrive, unless `hold` keeps the decode pending. */ +async function play(library: SoundLibrary, file = FILE, { hold = false } = {}): Promise { + library.play(file, 1, false); + await settle(); + if (hold) return; + for (const resolve of decodes.splice(0)) resolve(); + await settle(); +} + +describe('the sound library', () => { + it('fetches and decodes a file once, however often it is played', async () => { + const library = new SoundLibrary(); + await play(library); + await play(library); + expect(library.stats()).toMatchObject({ started: 2, decoded: 1, playing: 2 }); + // Two voices of one file, both running: plays overlap rather than replacing each other. + expect(sources.filter((s) => s.started)).toHaveLength(2); + }); + + it('remembers what it last started, so a muted check can see it at all', async () => { + const library = new SoundLibrary(); + library.play(FILE, 0.25, true); + await settle(); + for (const resolve of decodes.splice(0)) resolve(); + await settle(); + expect(library.stats().last).toEqual({ file: FILE, volume: 0.25, loop: true }); + }); + + it('a stop while the bytes are in flight keeps the sound from starting afterwards', async () => { + const library = new SoundLibrary(); + await play(library, FILE, { hold: true }); + library.stop(FILE); + for (const resolve of decodes.splice(0)) resolve(); + await settle(); + expect(library.stats()).toMatchObject({ started: 0, playing: 0 }); + }); + + it('but a stop before a play does not: an earlier silence is not a standing veto', async () => { + const library = new SoundLibrary(); + // Exactly what ending a run does – and the next run has to be able to play that sound again, + // even though its bytes are still unread at that point. + library.stop(''); + library.stop(FILE); + await play(library); + expect(library.stats()).toMatchObject({ started: 1, playing: 1 }); + }); + + it('stopping everything silences every voice, and only the named file otherwise', async () => { + const library = new SoundLibrary(); + await play(library, FILE); + await play(library, OTHER); + library.stop(FILE); + expect(library.stats()).toMatchObject({ playing: 1, stopped: 1 }); + + library.stopAll(); + expect(library.stats()).toMatchObject({ playing: 0, stopped: 2 }); + expect(sources.every((s) => s.stopped)).toBe(true); + }); + + it('a voice that ends by itself leaves the count, without being stopped', async () => { + const library = new SoundLibrary(); + await play(library); + sources[0]?.onended?.(); + expect(library.stats()).toMatchObject({ playing: 0, started: 1, stopped: 0 }); + }); + + it('a file that cannot be loaded fails once and is never asked for again', async () => { + const library = new SoundLibrary(); + vi.stubGlobal('fetch', () => Promise.resolve({ ok: false, status: 404 })); + vi.spyOn(console, 'error').mockImplementation(() => {}); + expect(await library.load(FILE)).toBeNull(); + expect(await library.load(FILE)).toBeNull(); + expect(library.stats()).toMatchObject({ failed: 1, started: 0 }); + + library.play(FILE, 1, false); + await settle(); + expect(library.stats().started).toBe(0); + }); +}); diff --git a/tools/e2e/21-list.mjs b/tools/e2e/21-list.mjs index 57e99b65..b497232c 100644 --- a/tools/e2e/21-list.mjs +++ b/tools/e2e/21-list.mjs @@ -89,7 +89,9 @@ export async function listTab(t) { check( 'a list of its own offers every kind, the struct included', (await json('__gm.list.info()')).offers.join(',') === - 'number,text,boolean,color,texture,statemachine,struct', + // In the order of the kind table (`core/properties/kinds.ts`) – a new kind is a row there, + // and `sound` (#45) is the newest of them. + 'number,text,boolean,color,texture,statemachine,sound,struct', JSON.stringify((await json('__gm.list.info()')).offers), ); await pickKind('Text'); diff --git a/tools/e2e/34-sound.mjs b/tools/e2e/34-sound.mjs new file mode 100644 index 00000000..07c8adfc --- /dev/null +++ b/tools/e2e/34-sound.mjs @@ -0,0 +1,275 @@ +import { edge, node } from '../lib/builders.mjs'; +import { LANES, MAKE_OBJECT, pickAssetType } from './lib/scaffold.mjs'; + +/** + * Sound assets (#45): uploaded like a typeface – picked from what the server holds through + * "+ → Sound", or dropped onto the editor – and played by a graph through `Play sound`. Nothing + * wears a sound and nothing places it; the asset says how loud it is and whether it loops, and the + * node scales the one and can overrule the other in either direction. + * + * Nothing is ever *heard* here: CI runs the browser muted (see tools/lib/browser.mjs). The witness + * is `__gm.sound.stats()` – voices started, stopped and in flight – the way `renderStats` is the + * witness for a repaint that did not happen. No pixel check could see any of this. + * + * The seed puts two generated WAVs on the server without making assets of them: a 0.25 s blip and + * a 0.75 s drone, told apart by their exact size rather than by their name. + */ +const LANE = LANES.sound; +const BLIP_BYTES = 4044; +const DRONE_BYTES = 12044; + +export async function sound(t) { + const { check, sleep, json, evaluate, mutate, click, clickText, press, play, stop, shot } = t; + const { select } = t; + const assetsOfType = (type) => + json(`__gm.state.project.assets.filter((a) => a.type === ${JSON.stringify(type)}) + .map((a) => ({ id: a.id, name: a.name, file: a.file, props: a.props }))`); + const stats = () => json('__gm.sound.stats()'); + /** A voice starts only once its bytes have been decoded, so what it changes is awaited. */ + const waitFor = async (read, done, tries = 25) => { + let value = await read(); + for (let i = 0; i < tries && !done(value); i++) { + await sleep(150); + value = await read(); + } + return value; + }; + + await press('Escape'); + await evaluate('document.activeElement instanceof HTMLElement && document.activeElement.blur()'); + await evaluate('__gm.state.select(null)'); + await evaluate("__gm.showTab('scene')"); + await sleep(200); + + // 34a. What the server holds, straight from the API: the two files the seed put there. + const stored = await evaluate(`fetch('/game-maker/api/index.php?r=sounds').then((r) => r.json())`); + const blip = stored.find((item) => item.bytes === BLIP_BYTES); + const drone = stored.find((item) => item.bytes === DRONE_BYTES); + check( + 'the server holds the two seeded sounds, named by their content', + Boolean(blip && drone) && /^[a-f0-9]{64}\.wav$/.test(blip.file), + JSON.stringify({ blip: blip?.file, drone: drone?.file, count: stored.length }), + ); + if (!blip || !drone) return; + check( + 'the listing carries the name each file was uploaded under', + blip.name === 'blip.wav' && drone.name === 'drone.wav', + JSON.stringify({ blip: blip.name, drone: drone.name }), + ); + + // 34b. "+ → Sound" offers them; a sound is picked rather than created empty, like a typeface. + await clickText('+'); + await sleep(200); + await pickAssetType(t, 'Sound'); + await sleep(500); + const picker = await json('__gm.soundPicker()'); + check( + 'the + menu opens the sound picker on what the server holds, none of it in the project yet', + Boolean(picker) && picker.files.includes(blip.file) && picker.used.length === 0, + JSON.stringify(picker), + ); + check( + 'a row nobody uses yet is captioned with the name the file was uploaded under', + picker.captions[picker.files.indexOf(blip.file)] === 'blip.wav', + JSON.stringify(picker.captions), + ); + await shot('34-sound-picker'); + + // 34c. The ▶ at a row's left is a preview: it plays that file without choosing it. + const playAt = await json(`__gm.soundPickerPlay(${JSON.stringify(drone.file)})`); + await click(playAt.x, playAt.y); + const previewed = await waitFor(stats, (s) => s.started > 0); + const afterPreview = await json('__gm.soundPicker()'); + check( + 'pressing a row’s ▶ plays that file and picks nothing', + previewed.started === 1 && + previewed.last?.file === drone.file && + afterPreview.played === drone.file && + afterPreview.selected === null, + JSON.stringify({ previewed, selected: afterPreview.selected, played: afterPreview.played }), + ); + + const row = await json(`__gm.soundPickerRow(${JSON.stringify(blip.file)})`); + await click(row.x, row.y); + await sleep(200); + await clickText('Use sound'); + await sleep(400); + const sounds = await assetsOfType('sound'); + check( + 'picking a row makes a sound asset of that file, named after the file rather than "Sound"', + sounds.length === 1 && sounds[0].file === blip.file && sounds[0].name === 'blip', + JSON.stringify(sounds), + ); + if (sounds.length !== 1) return; + const soundId = sounds[0].id; + check( + 'a fresh sound asset is full volume and one-shot', + sounds[0].props.volume === 1 && sounds[0].props.loop === false, + JSON.stringify(sounds[0].props), + ); + // Closing the picker silences its preview; nothing it started may outlive the dialog. + check( + 'closing the picker leaves nothing playing', + (await stats()).playing === 0, + JSON.stringify(await stats()), + ); + + // 34d. Its inspector: the two rows, and the preview that is the only way to tell which sound it is. + await select({ kind: 'asset', assetId: soundId }); + await sleep(300); + const volumeRow = await json("__gm.inspectorField('Volume', true)"); + const loopRow = await json("__gm.inspectorField('Loop', true)"); + check( + 'a sound asset has a Volume row and a Loop row', + Boolean(volumeRow) && Boolean(loopRow), + JSON.stringify({ volumeRow, loopRow }), + ); + await shot('34-sound-asset'); + + // 34e. A graph plays it. The behavior sits in its own lane; the node's volume is a factor on the + // asset's, so 0.5 × 0.4 has to reach the library as 0.2 rather than as either of the two. + await mutate(` + const p = __gm.state.project; + const scene = __gm.state.activeScene; + p.assets.find((a) => a.id === ${JSON.stringify(soundId)}).props.volume = 0.4; + const asset = structuredClone(p.assets.find((a) => a.id === 'mover')); + Object.assign(asset, { + id: 'sn-logic', + name: 'Noisy', + props: { enabled: true, runInEditor: false }, + variables: [], + graph: ${JSON.stringify({ + nodes: [ + node('sn-start', 'OnStart', 0, 0), + node('sn-play', 'PlaySound', 260, 0, { sound: soundId, volume: 0.5, loop: '' }), + ], + edges: [edge('sn-e1', 'sn-start', 'exec', 'sn-play', 'exec')], + })}, + }); + p.assets.unshift(asset); + ${MAKE_OBJECT} + scene.objects.push( + make('sn-box', 'crate', '', ${LANE}, 'Noisy box'), + make('sn-brain', 'sn-logic', 'sn-box', 0, 'Noisy'), + ); + __gm.lookAt(0, ${LANE});`); + await sleep(400); + + await evaluate('__gm.sound.stopAll()'); + const before = await stats(); + await play(); + const played = await waitFor(stats, (s) => s.started > before.started); + check( + 'a graph playing a sound reaches the library, at the asset volume scaled by the node’s', + played.started === before.started + 1 && + played.last?.file === blip.file && + Math.abs(played.last.volume - 0.2) < 0.001 && + played.last.loop === false, + JSON.stringify(played), + ); + await stop(); + await sleep(300); + + // 34f. A looping sound is music: it plays on, and ending the run ends it – nothing may outlive + // the world that started it. + await mutate(` + const p = __gm.state.project; + p.assets.find((a) => a.id === ${JSON.stringify(soundId)}).props.loop = true;`); + await sleep(300); + await play(); + const looping = await waitFor(stats, (s) => s.playing > 0 && s.last?.loop === true); + check( + 'the asset’s own loop flag reaches the library even though the node never asked for one', + looping.playing >= 1 && looping.last?.loop === true, + JSON.stringify(looping), + ); + // Well past the blip's own 0.25 s, so a one-shot would have ended by itself here. + await sleep(700); + const stillLooping = await stats(); + await stop(); + await sleep(400); + const afterStop = await stats(); + check( + 'a loop keeps playing while the run is on and ends with it', + stillLooping.playing >= 1 && afterStop.playing === 0, + JSON.stringify({ stillLooping, afterStop }), + ); + + // 34f-2. The node overrules the asset in both directions – the reason `loop` is a choice of three + // and not a checkbox: music has to be playable as a one-shot. + const setNodeLoop = (value) => + mutate(` + const p = __gm.state.project; + const graph = p.assets.find((a) => a.id === 'sn-logic').graph; + graph.nodes.find((n) => n.id === 'sn-play').params.loop = ${JSON.stringify(value)};`); + await setNodeLoop('false'); + await sleep(300); + const beforeOnce = await stats(); + await play(); + const once = await waitFor(stats, (s) => s.started > beforeOnce.started); + check('a node saying "off" plays a looping asset once', once.last?.loop === false, JSON.stringify(once)); + await stop(); + await sleep(300); + await setNodeLoop(''); + await sleep(300); + + // 34g. `Stop sound` silences it from inside the graph, without ending the run. + await mutate(` + const p = __gm.state.project; + const graph = p.assets.find((a) => a.id === 'sn-logic').graph; + graph.nodes.push(${JSON.stringify(node('sn-stop', 'StopSound', 520, 0, { sound: soundId }))}); + graph.nodes.push(${JSON.stringify(node('sn-key', 'OnKeyDown', 260, 220, { key: 's' }))}); + graph.edges.push(${JSON.stringify(edge('sn-e2', 'sn-key', 'exec', 'sn-stop', 'exec'))});`); + await sleep(300); + await play(); + await waitFor(stats, (s) => s.playing > 0); + await press('s'); + const silenced = await waitFor(stats, (s) => s.playing === 0); + check( + 'Stop sound silences a looping sound while the run carries on', + silenced.playing === 0 && (await json('__gm.playInfo()')).playing === true, + JSON.stringify(silenced), + ); + await stop(); + await sleep(300); + + // 34h. Dropping a sound file onto the editor uploads it and makes an asset named after the file. + const dropped = await evaluate(` + (async () => { + const bytes = await fetch('/game-maker/data/sounds/${drone.file}').then((r) => r.arrayBuffer()); + const file = new File([bytes], 'drone.wav', { type: 'audio/wav' }); + const data = new DataTransfer(); + data.items.add(file); + const canvas = document.querySelector('canvas'); + canvas.dispatchEvent(new DragEvent('drop', { dataTransfer: data, bubbles: true, cancelable: true })); + return file.size; + })()`); + await waitFor( + () => assetsOfType('sound'), + (list) => list.length === 2, + ); + const both = await assetsOfType('sound'); + check( + 'a dropped sound is uploaded and becomes an asset named after the file', + dropped === DRONE_BYTES && + both.length === 2 && + both.some((a) => a.file === drone.file && a.name === 'drone'), + JSON.stringify({ dropped, both, notice: await json('__gm.state.notice') }), + ); + + // 34i. The step takes back everything it made, so the steps after it see the fixture again. + await mutate(` + const gone = new Set(['sn-logic', ...${JSON.stringify(both.map((a) => a.id))}]); + const project = __gm.state.project; + project.assets = project.assets.filter((a) => !gone.has(a.id)); + __gm.state.activeScene.objects = __gm.state.activeScene.objects.filter( + (o) => !['sn-box', 'sn-brain'].includes(o.id)); + __gm.state.select(null); + __gm.lookAt(0, 0);`); + await sleep(300); + check( + 'the step leaves nothing of its own behind', + (await assetsOfType('sound')).length === 0 && (await stats()).playing === 0, + JSON.stringify(await assetsOfType('sound')), + ); +} diff --git a/tools/e2e/index.mjs b/tools/e2e/index.mjs index 1bd7ba7f..69ad322f 100644 --- a/tools/e2e/index.mjs +++ b/tools/e2e/index.mjs @@ -36,6 +36,7 @@ import { font } from './30-font.mjs'; import { particles } from './31-particles.mjs'; import { scope } from './32-scope.mjs'; import { renderOnChange } from './33-render-on-change.mjs'; +import { sound } from './34-sound.mjs'; export const STEPS = [ hierarchy, @@ -71,4 +72,5 @@ export const STEPS = [ particles, scope, renderOnChange, + sound, ]; diff --git a/tools/e2e/lib/scaffold.mjs b/tools/e2e/lib/scaffold.mjs index eae48742..7929186b 100644 --- a/tools/e2e/lib/scaffold.mjs +++ b/tools/e2e/lib/scaffold.mjs @@ -19,6 +19,7 @@ export const LANES = { transform: -1500, font: -1900, particles: -2300, + sound: -2700, }; /** diff --git a/tools/lib/audio.mjs b/tools/lib/audio.mjs new file mode 100644 index 00000000..ec025b64 --- /dev/null +++ b/tools/lib/audio.mjs @@ -0,0 +1,47 @@ +/** + * The seed's sounds, built here instead of shipped or taken from the system: unlike the DejaVu + * typefaces there is no audio file the host and the CI image are guaranteed to share, and a binary + * in the repo would be a binary in the repo. WAV is the one format that can be written by hand. + * + * The bytes are a pure function of the arguments, so storage stays content-addressed: re-seeding + * writes the same hash and the checks can name a file by its exact size. + */ + +const SAMPLE_RATE = 8000; +const BITS = 16; +const CHANNELS = 1; + +/** + * A mono 16-bit PCM sine, faded in and out over a millisecond so it neither clicks on start nor on + * stop – a decoder is happy either way, but a human listening to the editor's preview is not. + */ +export function sineWav(seconds, frequency = 440) { + const frames = Math.round(SAMPLE_RATE * seconds); + const data = Buffer.alloc(frames * 2); + const fade = Math.max(1, Math.round(SAMPLE_RATE * 0.001)); + for (let i = 0; i < frames; i++) { + const envelope = Math.min(1, i / fade, (frames - 1 - i) / fade); + const value = Math.sin((2 * Math.PI * frequency * i) / SAMPLE_RATE) * envelope * 0.6; + data.writeInt16LE(Math.round(value * 32767), i * 2); + } + return withRiffHeader(data); +} + +function withRiffHeader(data) { + const byteRate = (SAMPLE_RATE * CHANNELS * BITS) / 8; + const header = Buffer.alloc(44); + header.write('RIFF', 0); + header.writeUInt32LE(36 + data.length, 4); + header.write('WAVE', 8); + header.write('fmt ', 12); + header.writeUInt32LE(16, 16); // PCM header length + header.writeUInt16LE(1, 20); // 1 = uncompressed PCM + header.writeUInt16LE(CHANNELS, 22); + header.writeUInt32LE(SAMPLE_RATE, 24); + header.writeUInt32LE(byteRate, 28); + header.writeUInt16LE((CHANNELS * BITS) / 8, 32); // block align + header.writeUInt16LE(BITS, 34); + header.write('data', 36); + header.writeUInt32LE(data.length, 40); + return Buffer.concat([header, data]); +} diff --git a/tools/lib/browser.mjs b/tools/lib/browser.mjs index fbdb4200..73653c46 100644 --- a/tools/lib/browser.mjs +++ b/tools/lib/browser.mjs @@ -16,7 +16,11 @@ const FLAGS = [ '--disable-sync', '--disable-extensions', '--no-first-run', + // Nothing is meant to be heard here – but the sounds still have to *run*, or the sound checks + // would be measuring a browser that refuses to start a voice rather than an editor that never + // asked for one. Muted plus autoplay-allowed is exactly that: it plays, silently. '--mute-audio', + '--autoplay-policy=no-user-gesture-required', ]; /** Starts chromium with the CDP port open; `proc.stderr` carries its last words. */ diff --git a/tools/lib/projects.mjs b/tools/lib/projects.mjs index 74e7be88..334008d2 100644 --- a/tools/lib/projects.mjs +++ b/tools/lib/projects.mjs @@ -107,6 +107,27 @@ export async function uploadFont(baseUrl, filePath) { return { file: result.file, name: result.name }; } +/** + * Uploads a sound and returns `{file, name}` for a sound asset. Takes the bytes rather than a path: + * the seed builds its WAVs itself (see `lib/audio.mjs`), because no audio file is guaranteed to + * exist on both the host and in the CI image. Content-addressed like a font (`data/sounds/` is + * git-ignored), so re-seeding the same bytes is a no-op. + */ +export async function uploadSound(baseUrl, bytes, fileName, type = 'audio/wav') { + const res = await fetch(`${baseUrl}api/index.php?r=sounds&name=${encodeURIComponent(fileName)}`, { + method: 'POST', + headers: { 'Content-Type': type }, + body: bytes, + signal: AbortSignal.timeout(20_000), + }); + const result = await res.json(); + if (!res.ok) { + console.error('sound upload failed', res.status, result); + process.exit(1); + } + return { file: result.file, name: result.name, bytes: result.bytes }; +} + /** Overwrites a project, keeping the name it already has on the server. */ export async function writeProject(projectId, baseUrl, project) { const url = `${baseUrl}api/index.php?r=projects/${projectId}`; diff --git a/tools/seed-e2e.mjs b/tools/seed-e2e.mjs index 8b58065d..f4cb8cec 100644 --- a/tools/seed-e2e.mjs +++ b/tools/seed-e2e.mjs @@ -6,8 +6,9 @@ * * Usage: node tools/seed-e2e.mjs [projectId] [baseUrl] */ +import { sineWav } from './lib/audio.mjs'; import { edge, node, port, sceneObject, variable } from './lib/builders.mjs'; -import { PROJECTS, cliArgs, uploadFont, writeProject } from './lib/projects.mjs'; +import { PROJECTS, cliArgs, uploadFont, uploadSound, writeProject } from './lib/projects.mjs'; import { adjustColorsShader } from './lib/shaders.mjs'; const { projectId, baseUrl } = cliArgs('seed-e2e.mjs', PROJECTS.fixture); @@ -22,6 +23,15 @@ const SYSTEM_FONTS = '/usr/share/fonts/truetype/dejavu'; await uploadFont(baseUrl, `${SYSTEM_FONTS}/DejaVuSerif.ttf`); await uploadFont(baseUrl, `${SYSTEM_FONTS}/DejaVuSansMono.ttf`); +/** + * Two sounds for the sound step to pick and to drop (#45), put on the server without being made + * into assets, exactly as the typefaces are. They are generated rather than read: nothing + * guarantees an audio file on both the host and in the CI image, and the two differ in length so a + * check can tell which one it got without listening – the browser runs muted. + */ +await uploadSound(baseUrl, sineWav(0.25, 880), 'blip.wav'); +await uploadSound(baseUrl, sineWav(0.75, 220), 'drone.wav'); + /** * "Mover" in five nodes: every frame, move the owner along the normalized arrow-key axis, * scaled by the `speed` variable per second (On update's dt feeds the dt input). "Collide with