import { findAssetOf } from '../../core/model/Project'; import { projectFiles } from '../../core/model/projectFiles'; import { clipsOf, cutChannels, cutRange, findSound, parseCutName } from '../../core/model/sound'; import { STREAM_RATE } from '../../core/music/streamQueue'; import { clipCopyTable, playedPlan, streamTable } from '../../core/music/streams'; import { playedSoundOf } from '../../core/properties/playedSound'; import { soundLibrary } from '../../viewport/SoundLibrary'; import { centreOf, type HookContext } from './hookHelpers'; /** The Sound tab (#343). */ export function soundHooks({ state, editor }: HookContext) { const { soundEditor } = editor; return { soundTab: { current: () => soundEditor.currentAssetId, info: () => soundEditor.shownInfo, rect: () => soundEditor.canvasRect(), /** Where a second of the file is on screen, at the middle of the wave. */ at: (seconds: number) => soundEditor.canvas.secondsToScreen(seconds), /** The stretch between the handles, in seconds; null while the file is not there. */ span: () => soundEditor.canvas.shownSpan(), duration: () => soundEditor.canvas.fileDuration(), /** The height zoom (#357): the factor, the middle of the level scale, and the top of a full-scale wave. */ level: () => soundEditor.canvas.levelZoom, scale: () => soundEditor.canvas.scaleCentre(), fullScaleTop: () => soundEditor.canvas.fullScaleTop(), /** The time zoom, and whether a zoom is on its way (#377). */ zoom: () => soundEditor.canvas.zoomInfo(), fitButton: () => centreOf(soundEditor.fitButton), cursor: () => soundEditor.canvas.cursor, playhead: () => soundEditor.canvas.playhead, playing: () => soundEditor.isPlaying, playButton: () => ({ ...centreOf(soundEditor.playButton), text: soundEditor.playButton.text }), reverseButton: () => ({ ...centreOf(soundEditor.reverseButton), on: soundEditor.reverseButton.toggled, enabled: soundEditor.reverseButton.enabled, }), /** The clips of the sound shown (#357): id, name and stored values; the picked one's id. */ clips: () => { const asset = findAssetOf(state.project, soundEditor.currentAssetId, 'sound'); return asset ? clipsOf(asset).map((c) => ({ id: c.id, name: c.name, props: { ...c.props } })) : []; }, pickedClip: () => soundEditor.canvas.pickedClip()?.id ?? null, /** The middle of the clip strip at a second of the file, on screen. */ stripAt: (seconds: number) => soundEditor.canvas.stripAt(seconds), /** The clips drawn faint over the wave, by name: all but the picked one. */ faintClips: () => soundEditor.canvas.faintClips(), /** Wave draws, its shift from where it was drawn, and ruler labels given a text (#360). */ drawInfo: () => soundEditor.canvas.drawInfo(), fadeInField: () => ({ ...centreOf(soundEditor.fadeInField), text: soundEditor.fadeInField.value }), fadeOutField: () => ({ ...centreOf(soundEditor.fadeOutField), text: soundEditor.fadeOutField.value }), /** The curve dropdowns show a picture of their curve; the tooltip names it. */ curveInButton: () => ({ ...centreOf(soundEditor.curveInButton), tooltip: soundEditor.curveInButton.tooltipText(), }), curveOutButton: () => ({ ...centreOf(soundEditor.curveOutButton), tooltip: soundEditor.curveOutButton.tooltipText(), }), /** What a sound or a clip of one is played by: its file, or the name of the clip's cut. */ played: (id: string) => { const ref = findSound(state.project, id); return ref ? playedSoundOf(ref) : null; }, /** The samples a name plays, cut and faded: their count, rate, and the first and last few of channel 0. */ samples: async (name: string) => { const buffer = await soundLibrary.load(name); if (!buffer) return null; const data = buffer.getChannelData(0); return { length: buffer.length, rate: buffer.sampleRate, head: Array.from(data.slice(0, 4)), tail: Array.from(data.slice(-4)), }; }, }, /** Streamed sounds (#349). */ streams: { /** Done once every copy wanted has been made. */ idle: () => editor.streamKeeper.idle(), /** Every name a game streams, with its copy. */ table: () => [...streamTable(state.project)].map(([name, copy]) => ({ name, ...copy })), /** * Streams `seconds` of a name offline from `offset` seconds in, heard from `at`, looped or not, * and holds it against its copy decoded whole: the largest difference of any sample from what * should be heard there – silence before `at` – against the feed's own decoder and against the * browser's `decodeAudioData`, and how many samples came too late. */ exact: async (name: string, offset: number, loop: boolean, at: number, seconds: number) => { const entry = streamTable(state.project).get(name); if (!entry) return null; const plan = playedPlan(entry.frames, offset, loop); const frames = Math.round((at + seconds) * STREAM_RATE); const [played, whole, browser] = await Promise.all([ soundLibrary.renderStream(name, plan, at, frames), soundLibrary.streamedWhole(name), soundLibrary.decodeApart(entry.file, STREAM_RATE), ]); if (!played || !whole) return null; const first = Math.round(at * STREAM_RATE); const placeOf = (i: number) => loop ? (plan.from + i - first) % entry.frames : plan.from + i - first; const worst = (reference: (c: number, place: number) => number) => { let most = 0; played.channels.forEach((side, c) => { for (let i = 0; i < frames; i++) { const place = placeOf(i); const expected = i < first || place >= entry.frames ? 0 : reference(c, place); most = Math.max(most, Math.abs(side[i]! - expected)); } }); return most; }; return { frames: entry.frames, decoded: browser.length, own: worst((c, place) => whole[c]![place]!), browser: worst( (c, place) => browser.getChannelData(Math.min(c, browser.numberOfChannels - 1))[place]!, ), skipped: played.skipped, }; }, /** * What a game decodes that has just opened the project, in a library of its own so nothing this * page holds counts: the names it decodes ahead, whether that decoded the whole file of a name's * sound, and what the name then plays held against its cut made from that file – samples, rate, * the loudest difference of a sample and of the level of any 20 ms. */ fresh: async (name: string) => { const library = soundLibrary.aside(); library.setStreams(streamTable(state.project)); library.setCopies(clipCopyTable(state.project)); const ahead = projectFiles(state.project).sounds; await Promise.all(ahead.map((file) => library.load(file))); const { file, cut } = parseCutName(name); const whole = library.decoded(file) !== null; const played = await library.load(name); const original = await soundLibrary.decodeApart(file, STREAM_RATE); if (!played || !cut || played.sampleRate !== STREAM_RATE) return { ahead, whole, played: null }; const [reference] = cutChannels( [original.getChannelData(0)], cutRange(original.length, STREAM_RATE, cut), ); const heard = played.getChannelData(0); let worst = 0; let level = 0; const span = Math.round(0.02 * STREAM_RATE); for (let from = 0; from + span <= heard.length; from += span) { let own = 0; let wanted = 0; for (let i = from; i < from + span; i++) { worst = Math.max(worst, Math.abs(heard[i]! - (reference![i] ?? 0))); own += heard[i]! ** 2; wanted += (reference![i] ?? 0) ** 2; } // Only where it is heard: the fades start from silence. if (wanted / span > 1e-6) level = Math.max(level, Math.abs(10 * Math.log10(own / wanted))); } return { ahead, whole, played: { length: played.length, wanted: reference!.length, worst, level }, decoded: library.stats().decoded, }; }, /** What a sound and each of its clips keep of their copies. */ copies: (id: string) => { const asset = findAssetOf(state.project, id, 'sound'); if (!asset) return null; return { sound: asset.stream ?? null, clips: clipsOf(asset).map((clip) => ({ id: clip.id, stream: clip.stream ?? null })), }; }, }, }; }