import { execFileSync } from 'node:child_process'; import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { uploadVideo } from '../lib/projects.mjs'; import { MAKE_OBJECT } from './lib/scaffold.mjs'; /** * A composition placed in the scene (#417). In the editor the placement shows the composition's * first frame, and a click on that picture picks the placement. Played, a behavior below it starts * it: each item becomes an object below the placement while it lasts, the first track in front, a * clip moves the item it overlaps, and `On composition finished` hears the end, by which time no * item is left. Nothing of the run stays in the scene. An item's video is waited for: the clock * stands until the Game tab has its first picture, for a clip that starts later the picture there, * and then the video plays at once. A recording that runs meanwhile takes no frame. When the item * ends, its video stops and goes quiet with it. A sound item is planned once the clock runs, timed * to the cut. Recorded not in real time from a slowed editor, the file still has every frame, the * video's picture at each, and its sound and the sound item's in step. Behind another tab, where the * page draws no frames, a recording goes on all the same, not in real time and in real time. A video * item that begins there is in it from its first frame, and switched between the tabs while it * records, its video still turns in step. * * Everything it adds carries a `cp68` id and goes again at the end. */ const PLACE = 'cp68-place'; const DONE = 'cut 68 done'; const node = (id, type, x, y, params = {}) => ({ id: `cp68-${id}`, type, x, y, params }); const link = (id, from, fromPort, to, toPort) => ({ id: `cp68-${id}`, from: { node: `cp68-${from}`, port: fromPort }, to: { node: `cp68-${to}`, port: toPort }, }); export async function composition(t) { const { check, sleep, json, evaluate, mutate, press, play, stop, showTab, waitUntil } = t; await showTab('scene'); const graph = { nodes: [ node('start', 'OnStart', 0, 0), node('owner', 'Owner', 0, 150), node('play', 'PlayComposition', 300, 0), node('finished', 'OnCompositionFinished', 0, 300, { composition: 'cp68-cut' }), node('log', 'Log', 300, 300, { message: DONE }), ], edges: [ link('e1', 'start', 'exec', 'play', 'exec'), link('e2', 'owner', 'object', 'play', 'object'), link('e3', 'finished', 'exec', 'log', 'exec'), ], }; // Red from the start, a larger blue behind it from the next frame – built after red, it still goes // behind – and green after them, moved by a clip; ten frames a second, so the cut lasts 1.2 s. // Placed where the camera looks, in front of what is there. await mutate(` const p = __gm.state.project; const scene = __gm.state.activeScene; const cam = scene.objects.find((o) => o.id === 'o3').position; ${MAKE_OBJECT} const sprite = p.assets.find((a) => a.type === 'sprite'); const look = (o, size, color) => { o.overrides = { width: size, height: size, color, texture: '' }; return o; }; const red = look(make('cp68-red', sprite.id, '', 0, 'Red 68'), 200, 0xff0000); const blue = look(make('cp68-blue', sprite.id, '', 0, 'Blue 68'), 300, 0x0000ff); const green = look(make('cp68-green', sprite.id, '', 0, 'Green 68'), 200, 0x00ff00); for (const o of [red, blue, green]) delete o.groupId; p.assets.push( { id: 'cp68-cut', type: 'composition', name: 'Cut 68', props: { fps: 10, loop: false }, objects: [red, blue, green], groups: [], tracks: [{ id: 'cp68-front', name: 'Front' }, { id: 'cp68-back', name: 'Back' }], items: [ { id: 'cp68-red', track: 'cp68-front', start: 0, length: 6, offset: 0 }, { id: 'cp68-blue', track: 'cp68-back', start: 1, length: 5, offset: 0 }, { id: 'cp68-green', track: 'cp68-front', start: 6, length: 6, offset: 0 }, ], clips: [{ id: 'cp68-clip', track: 'cp68-front', animation: 'cp68-move', start: 6 }], }, { id: 'cp68-move', type: 'animation', name: 'Slide 68', props: { fps: 10, playback: 'once' }, root: sprite.id, tracks: [{ id: 'cp68-row', target: '', property: 'x', keys: [{ frame: 0, value: 0 }, { frame: 5, value: 100 }] }], }, { id: 'cp68-director', type: 'behavior', name: 'Director 68', props: { enabled: true, runInEditor: false }, variables: [], graph: ${JSON.stringify(graph)} }, ); const place = make('${PLACE}', 'cp68-cut', '', 0, 'Cut 68'); Object.assign(place.position, { x: cam.x, y: cam.y, z: 2000 }); scene.objects.push(place, make('cp68-logic', 'cp68-director', '${PLACE}', 0, 'Director 68')); __gm.lookAt(cam.x, cam.y); `); const id = (item) => `${PLACE}~cp68-${item}`; // 68a. In the editor the placement shows its first frame – red alone – and a click on the picture // picks the placement, since what it shows is edited in the composition. const order = await json('__gm.drawOrder()'); // Off the centre, where the placement's own symbol sits. const rect = await json(`__gm.objectRect(${JSON.stringify(id('red'))})`); const stack = rect ? await json(`__gm.sceneStack(${rect.x + rect.w * 0.35}, ${rect.y + rect.h * 0.35})`) : null; check( 'a placed composition shows its first frame, and a click on it picks the placement', order.includes(id('red')) && !order.includes(id('blue')) && !order.includes(id('green')) && JSON.stringify(stack) === JSON.stringify([PLACE]), JSON.stringify({ order: order.filter((o) => o.startsWith('cp68')), stack }), ); // 68b. Played: the items come and go below the placement, sampled every frame of the editor. await play(); const seen = await evaluate(`new Promise((resolve) => { const frames = []; const sample = () => { const world = __gm.world(); if (!world) return resolve(JSON.stringify(frames)); const items = world.entities.filter((e) => e.objectId.startsWith('${PLACE}~')); const green = items.find((e) => e.objectId === '${id('green')}'); frames.push({ ids: items.map((e) => e.objectId.slice('${PLACE}~cp68-'.length)), parents: [...new Set(items.map((e) => e.parent))], greenX: green ? green.x : null, done: world.log.includes('${DONE}'), }); if (frames.length > 400 || frames.at(-1).done) return resolve(JSON.stringify(frames)); requestAnimationFrame(sample); }; sample(); })`); const frames = JSON.parse(seen); const both = frames.find((f) => f.ids.length === 2); const greenXs = frames.map((f) => f.greenX).filter((x) => x !== null); check( 'Play composition builds the items below the placement, the first track in front, one after the other', both !== undefined && JSON.stringify(both.ids) === JSON.stringify(['blue', 'red']) && both.parents.length === 1 && both.parents[0] === PLACE && !frames.some((f) => f.ids.join('+') === 'red+blue') && frames.some((f) => JSON.stringify(f.ids) === JSON.stringify(['green'])) && !frames.some((f) => f.ids.includes('green') && f.ids.includes('red')), JSON.stringify(frames.filter((_, i) => i % 5 === 0).map((f) => f.ids.join('+'))), ); check( 'a clip moves the item it overlaps, from its value on', greenXs.length >= 3 && greenXs[0] < 40 && greenXs.at(-1) > greenXs[0] && greenXs.every((x, i) => i === 0 || x >= greenXs[i - 1]), JSON.stringify(greenXs), ); const last = frames.at(-1); check( 'On composition finished hears the end, by which time no item is left', last?.done === true && last.ids.length === 0, JSON.stringify(last), ); await sleep(300); await stop(); // 68c. The run's objects stay in the run: the scene holds the placement and its behavior only. const stored = await json( `__gm.state.activeScene.objects.filter((o) => o.id.startsWith('cp68')).map((o) => o.id).sort()`, ); check( 'nothing of a played composition stays in the scene', JSON.stringify(stored) === JSON.stringify(['cp68-logic', PLACE].sort()), JSON.stringify(stored), ); // 68d. A video item at frame 0, played from On start: the composition's clock stands while the Game // tab loads the video, so its first frames are seen – it ran on without them before. const uploaded = await uploadVideo( t.baseUrl, readFileSync(new URL('./fixtures/colors.webm', import.meta.url)), 'colors.webm', ); await mutate(` const p = __gm.state.project; p.assets.push( { id: 'cp68-video', type: 'video', name: 'Colors 68', props: {}, file: ${JSON.stringify(uploaded.file)}, width: 320, height: 180, duration: 2, fps: 30, audio: true }, { id: 'cp68-screen', type: 'sprite', name: 'Screen 68', props: { width: 160, height: 90, color: 0xffffff, texture: 'cp68-video' } }, ); const c = p.assets.find((a) => a.id === 'cp68-cut'); // Sixty frames a second: in steps of a tenth, a clock that ran on for a few frames still read 0. c.props.fps = 60; c.objects = [{ ...structuredClone(c.objects[0]), id: 'cp68-v', assetId: 'cp68-screen', name: 'Video 68', overrides: {} }]; c.items = [{ id: 'cp68-v', track: 'cp68-front', start: 0, length: 120, offset: 0 }]; c.clips = [];`); // The video time the composition writes into its item every frame. const clock = `__gm.world()?.entities.find((e) => e.objectId === '${id('v')}')?.props.videoTime ?? null`; // Sampled from before the click on Play: play() settles after the start, and a video that loads // quickly is ready by then, with no wait left to see (CI task 669). await evaluate(`{ const loading = []; const until = performance.now() + 20000; window.cp68Wait = new Promise((done) => { const step = () => { const playing = __gm.state.mode === 'play'; const time = playing ? ${clock} : null; const player = playing ? __gm.gameVideo('${id('v')}') : null; if (player?.ready) return done(JSON.stringify({ loading, ready: time })); if (player) loading.push(time); if (performance.now() > until) return done(JSON.stringify({ loading, ready: null })); requestAnimationFrame(step); }; step(); }); // Not the promise as the block's value: evaluate would wait for it, and Play would come too late. null; }`); await play(); const wait = JSON.parse(await evaluate('window.cp68Wait')); await sleep(500); const after = await json(clock); check( 'a played composition waits for the first picture of an item’s video, then runs on', wait.loading.length > 0 && wait.loading.every((time) => time === 0) && wait.ready !== null && wait.ready === 0 && after > 0.2, JSON.stringify({ frames: wait.loading.length, loading: wait.loading.slice(0, 8), ready: wait.ready, after, }), ); await stop(); // 68e. The item wears a clip that starts at 0.8 s: its player loads, then seeks back to 0.8. The // clock stands until the Game tab draws that picture, and from its first step on the video // plays – it was held and sought frame by frame for 0.1 s before, its first pictures stuck. await mutate(` const p = __gm.state.project; p.assets.find((a) => a.id === 'cp68-video').clips = [{ id: 'cp68-late', name: 'Late', props: { start: 0.8, end: 0 } }]; p.assets.find((a) => a.id === 'cp68-screen').props.texture = 'cp68-late';`); await play(); const rows = JSON.parse( await evaluate(`new Promise((done) => { const rows = []; const step = () => { const player = __gm.gameVideo('${id('v')}'); rows.push({ time: ${clock}, drawn: __gm.gameDrawnRect('${id('v')}') !== null, at: player?.time ?? null, paused: player?.paused ?? null }); if (rows.length >= 240 || !__gm.world()) return done(JSON.stringify(rows)); requestAnimationFrame(step); }; step(); })`), ); const first = rows.findIndex((row) => row.drawn); const moving = rows.filter((row) => row.drawn && row.time > 0).slice(0, 3); check( 'a clip that starts later is waited for, then plays from its first step on', first >= 0 && rows.slice(0, first).every((row) => row.time === 0) && Math.abs(rows[first].at - 0.8) < 0.02 && moving.length === 3 && moving.every((row) => row.paused === false), JSON.stringify({ first, shown: rows[first], moving }), ); await stop(); // 68f. A recording runs when Play composition starts the cut: no frame is taken while the Game // tab leaves the item's video out – it recorded the empty frames of the wait before. A clip // that starts elsewhere, so a new player loads: the one of 68e is still there, at its picture. await mutate(` const video = __gm.state.project.assets.find((a) => a.id === 'cp68-video'); video.clips.push({ id: 'cp68-later', name: 'Later', props: { start: 1.2, end: 0 } }); __gm.state.project.assets.find((a) => a.id === 'cp68-screen').props.texture = 'cp68-later';`); const flow = structuredClone(graph); flow.nodes.push( node('record', 'StartRecording', 300, -150, { name: 'cp68 take', resolution: 'view', fps: '60' }), node('key', 'OnKeyDown', 0, 450, { key: 'p' }), ); flow.edges = flow.edges.filter((e) => e.id !== 'cp68-e1'); flow.edges.push(link('e4', 'start', 'exec', 'record', 'exec'), link('e5', 'key', 'exec', 'play', 'exec')); await mutate( `__gm.state.project.assets.find((a) => a.id === 'cp68-director').graph = ${JSON.stringify(flow)};`, ); const { uploads, takes } = await json('__gm.recording()'); const take = `__gm.recording().takes[${takes.length}]`; await play(); await waitUntil(`${take}?.state === 'recording' && ${take}.lastFrameAt !== null`, { label: 'the recording takes frames', }); // Sampled every frame in the page, from before the key on. await evaluate(`{ const rows = (window.cp68Rows = []); const step = () => { const exists = (__gm.world()?.entities ?? []).some((e) => e.objectId === '${id('v')}'); rows.push({ exists, drawn: __gm.gameDrawnRect('${id('v')}') !== null, taken: ${take}?.lastFrameAt ?? null }); const drawn = rows.findIndex((row) => row.drawn); if ((drawn < 0 || rows.length <= drawn + 20) && rows.length < 400) requestAnimationFrame(step); }; requestAnimationFrame(step); }`); await press('p'); await waitUntil( `window.cp68Rows.length >= 400 || (window.cp68Rows.some((row) => row.drawn) && window.cp68Rows.length > window.cp68Rows.findIndex((row) => row.drawn) + 20)`, { label: 'the item is drawn' }, ); const takeRows = await json('window.cp68Rows'); await stop(); // The frames in which the item was there and its video still left out. const missing = takeRows.filter((row) => row.exists && !row.drawn); const shown = takeRows.findIndex((row) => row.drawn); check( 'a recording takes no frame while a composition waits for its video, and goes on after', missing.length >= 2 && missing.every((row) => row.taken === missing[0].taken) && shown >= 0 && takeRows.at(-1).taken > missing.at(-1).taken, JSON.stringify({ missing: missing.length, takenWhileMissing: new Set(missing.map((row) => row.taken)).size - 1, shown, after: takeRows.at(-1).taken !== null && missing.length > 0 ? Math.round(takeRows.at(-1).taken - missing.at(-1).taken) : null, }), ); await waitUntil( `__gm.recording().uploads.length > ${uploads.length} && ['added', 'failed'].includes(__gm.recording().uploads.at(-1).state)`, { timeout: 60_000, label: 'the recording is in the project' }, ); const madeId = (await json('__gm.recording()')).uploads.at(-1).assetId ?? ''; // 68g. The item ends after half a second, its video of two seconds with sound in the middle: the // item's player stops and goes quiet at once – it played on, heard, until it was let go 2-4 s // later. await mutate(` const p = __gm.state.project; p.assets.find((a) => a.id === 'cp68-director').graph = ${JSON.stringify(graph)}; p.assets.find((a) => a.id === 'cp68-screen').props.texture = 'cp68-video'; p.assets.find((a) => a.id === 'cp68-cut').items[0].length = 30;`); await play(); const cut = JSON.parse( await evaluate(`new Promise((done) => { const rows = []; let gone = -1; const step = () => { const exists = (__gm.world()?.entities ?? []).some((e) => e.objectId === '${id('v')}'); const player = __gm.gameVideo('${id('v')}'); rows.push({ exists, paused: player?.paused ?? null, muted: player?.muted ?? null }); if (gone < 0 && rows.some((row) => row.exists) && !exists) gone = rows.length - 1; if ((gone >= 0 && rows.length > gone + 30) || rows.length >= 600 || !__gm.world()) return done(JSON.stringify({ gone, heard: rows.filter((row) => row.exists && row.paused === false && row.muted === false).length, after: rows.slice(gone + 12) })); requestAnimationFrame(step); }; step(); })`), ); await stop(); check( 'the video of an item that ends stops and goes quiet with it', cut.gone > 0 && cut.heard > 0 && cut.after.length > 0 && cut.after.every((row) => row.paused === true && row.muted === true), JSON.stringify({ gone: cut.gone, heard: cut.heard, after: cut.after.slice(0, 3), last: cut.after.at(-1), }), ); // 68h (#418). A sound item half a second in, beside the video item from 68d: the cut waits for the // video, and its sound with it – planned once the clock runs, for the second of the audio // clock the cut reaches the item at, and to its end. const sounds = await evaluate(`fetch('/feuerwerk/api/index.php?r=sounds').then((r) => r.json())`); const blip = sounds.find((item) => item.bytes === 4044); await mutate(` const p = __gm.state.project; p.assets.push({ id: 'cp68-blip', type: 'sound', name: 'Blip 68', props: {}, file: ${JSON.stringify(blip?.file ?? '')} }); p.assets.find((a) => a.id === 'cp68-screen').props.texture = 'cp68-video'; const c = p.assets.find((a) => a.id === 'cp68-cut'); c.items = [ { id: 'cp68-v', track: 'cp68-front', start: 0, length: 120, offset: 0 }, { id: 'cp68-tone', track: 'cp68-back', start: 30, length: 30, offset: 0, audio: { kind: 'sound', asset: 'cp68-blip', volume: 0, bus: '#composition' } }, ];`); await evaluate(`{ window.cp68Sound = new Promise((done) => { const rows = []; const until = performance.now() + 20000; const step = () => { if (__gm.state.mode === 'play') { const time = ${clock}; const tone = __gm.sound.playbacks().find((p) => p.name === 'composition:cp68-cut'); rows.push({ time, clock: __gm.sound.clock(), planned: tone?.clips.length ?? 0, clip: tone?.clips[0] ?? null }); if (time !== null && time >= 0.6) { const buses = __gm.routing.buses(); rows.at(-1).routes = { tone: buses.find((b) => b.name === 'play:' + tone?.id)?.into ?? null, cut: buses.find((b) => b.name === 'mix:cp68-cut')?.into ?? null, videos: __gm.routing.media().filter((m) => m.routed).map((m) => m.bus), }; return done(JSON.stringify(rows)); } } if (performance.now() > until) return done(JSON.stringify(rows)); requestAnimationFrame(step); }; step(); }); null; }`); await play(); const toneRows = JSON.parse(await evaluate('window.cp68Sound')); await stop(); const ran = toneRows.findIndex((row) => row.time > 0); const reached = toneRows.find((row) => row.time >= 0.5); const toneClip = toneRows.at(-1)?.clip; const early = reached && toneClip ? reached.clock - toneClip.at : null; check( 'a sound item is planned only once the cut’s clock runs, for when the cut reaches it, to its end', ran > 0 && toneRows.slice(0, ran).every((row) => row.planned === 0) && toneClip?.sound === 'cp68-blip' && toneClip.offset === 0 && Math.abs(toneClip.duration - 0.5) < 0.002 && early !== null && early > -0.02 && early < 0.05, JSON.stringify({ waited: ran, early, reached, toneClip }), ); // 68i. The sound item picked the composition's bus, the video item picked none: both go into the // composition's bus, and that one into the master. const routes = toneRows.at(-1)?.routes; check( 'a sound item and a video item go into the composition’s bus, and that one into the master', routes?.tone === 'mix:cp68-cut' && routes.cut === 'mix' && routes.videos.length > 0 && routes.videos.every((bus) => bus === 'mix:cp68-cut'), JSON.stringify(routes), ); // 68j (#423). Recorded not in real time while the editor draws a frame every 45 ms, about 20 a // second: the file still has every frame, 1/60 s apart. The video item covers the picture, // its colours turn every half second of its file and so every 30 frames of the recording, and // its sound is heard from the start. The blip item starts as green does. await mutate(` const p = __gm.state.project; p.assets.find((a) => a.id === 'cp68-screen').props = { width: 4000, height: 3000, color: 0xffffff, texture: 'cp68-video' }; const c = p.assets.find((a) => a.id === 'cp68-cut'); c.items = [ { id: 'cp68-v', track: 'cp68-front', start: 0, length: 120, offset: 0 }, { id: 'cp68-tone', track: 'cp68-back', start: 30, length: 30, offset: 0, audio: { kind: 'sound', asset: 'cp68-blip', volume: 0, bus: '#composition' } }, ]; p.assets.find((a) => a.id === 'cp68-director').graph = ${JSON.stringify({ nodes: [ node('start', 'OnStart', 0, 0), node('owner', 'Owner', 0, 150), node('record', 'StartRecording', 300, 0, { name: 'cp68 steady', resolution: '720p', fps: '60', convert: false, adopt: false, realTime: false, }), node('play', 'PlayComposition', 600, 0), node('finished', 'OnCompositionFinished', 0, 300, { composition: 'cp68-cut' }), node('stop', 'StopRecording', 300, 300), ], edges: [ link('e1', 'start', 'exec', 'record', 'exec'), link('e2', 'record', 'exec', 'play', 'exec'), link('e3', 'owner', 'object', 'play', 'object'), link('e4', 'finished', 'exec', 'stop', 'exec'), ], })};`); const before = await json('__gm.recording()'); const steadyTake = `__gm.recording().takes[${before.takes.length}]`; await evaluate(`(() => { window.__cp68slow = true; const slow = () => { if (!window.__cp68slow) return; const until = performance.now() + 45; while (performance.now() < until); requestAnimationFrame(slow); }; requestAnimationFrame(slow); })()`); await play(); await waitUntil(`${steadyTake}?.state === 'done' || ${steadyTake}?.state === 'failed'`, { timeout: 60_000, label: 'the recording not in real time is saved', }); await evaluate('window.__cp68slow = false; 0'); await stop(); await waitUntil( `__gm.recording().uploads.length > ${before.uploads.length} && ['kept', 'failed'].includes(__gm.recording().uploads.at(-1).state)`, { timeout: 30_000, label: 'the recording is uploaded' }, ); const steady = await json(steadyTake); const steadyUpload = (await json('__gm.recording()')).uploads.at(-1); const steadyLines = (await json('__gm.console.lines()')) .map((line) => line.text) .filter((text) => text.includes('"cp68 steady"')); /** * What ffmpeg reads from the original of a recording: its frames, where they are not 1/60 s * apart, where the video's colour turns, the video's tone and where the blip lies (from green). */ const readRecording = async (upload) => { const original = await fetch(`${t.baseUrl}data/videos/${upload.file.replace(/\.mp4$/, '.source.mkv')}`); const file = { frames: 0, gaps: [], turns: [], toneBefore: null, toneDuring: null, blipAt: null }; if (original.ok) { const dir = mkdtempSync(join(tmpdir(), 'cp68-')); const mkv = join(dir, 'steady.mkv'); writeFileSync(mkv, new Uint8Array(await original.arrayBuffer())); const pts = execFileSync('ffprobe', [ '-v', 'error', '-select_streams', 'v:0', '-show_entries', 'packet=pts_time', '-of', 'csv=p=0', mkv, ]) .toString() .trim() .split('\n') .map(Number) .sort((a, b) => a - b); file.frames = pts.length; file.gaps = pts.slice(1).flatMap((at, i) => (Math.abs(at - pts[i] - 1 / 60) > 0.0015 ? [i + 1] : [])); // Each frame as 8x4 pixels: the colour of the video item covering it, or of what lies behind. const pixels = execFileSync( 'ffmpeg', [ '-v', 'error', '-i', mkv, '-map', '0:v', '-vf', 'scale=8:4', '-f', 'rawvideo', '-pix_fmt', 'rgb24', '-', ], { maxBuffer: 64 * 1024 * 1024 }, ); const colours = []; for (let at = 0; at + 96 <= pixels.length; at += 96) { const [r, g, b] = [0, 1, 2].map((c) => pixels[at + 13 * 3 + c]); colours.push( r > 180 && g > 180 && b > 180 ? 'W' : r > 150 ? 'R' : g > 150 ? 'G' : b > 150 ? 'B' : '-', ); } file.turns = colours.flatMap((colour, i) => i === 0 || colour !== colours[i - 1] ? [`${i}${colour}`] : [], ); const sound = execFileSync( 'ffmpeg', ['-v', 'error', '-i', mkv, '-map', '0:a', '-ac', '1', '-ar', '48000', '-f', 'f32le', '-'], { maxBuffer: 64 * 1024 * 1024 }, ); const samples = new Float32Array(sound.buffer, sound.byteOffset, sound.length / 4); const rms = (from, to) => { let sum = 0; const a = Math.max(0, Math.round(from * 48000)); const b = Math.min(samples.length, Math.round(to * 48000)); for (let i = a; i < b; i++) sum += samples[i] * samples[i]; return b > a ? Math.sqrt(sum / (b - a)) : 0; }; const red = Number.parseInt(file.turns.find((turn) => turn.endsWith('R')) ?? '-1', 10); const green = Number.parseInt(file.turns.find((turn) => turn.endsWith('G')) ?? '-1', 10); if (red >= 0 && green > red) { file.toneBefore = red > 6 ? rms(0, (red - 3) / 60) : 0; file.toneDuring = rms((red + 3) / 60, (green - 3) / 60); // The blip is loud over the tone of the video: the first 2 ms that are. for (let at = red / 60; at < (green + 30) / 60; at += 0.002) if (rms(at, at + 0.002) > 3 * file.toneDuring + 0.05) { file.blipAt = Math.round((at - green / 60) * 1000); break; } } rmSync(dir, { recursive: true, force: true }); } // The video's four colours one after the other, whatever lay behind it before and after. const colourOrder = file.turns.map((turn) => turn.at(-1)).join(''); const turned = file.turns .slice(colourOrder.indexOf('RGBW'), colourOrder.indexOf('RGBW') + 4) .map((turn) => Number.parseInt(turn, 10)); return { ...file, ok: original.ok, status: original.status, colourOrder, turned }; }; const file = await readRecording(steadyUpload); const original = { ok: file.ok, status: file.status }; const { colourOrder, turned } = file; check( 'a recording not in real time has every frame, 1/60 s apart, though the editor drew about 20 a second', original.ok && steady.realTime === false && steady.failure === null && steady.skipped === 0 && steady.dropped === 0 && file.frames === steady.frames && file.frames >= 120 && file.gaps.length === 0 && steady.tookMs > steady.durationMs * 1.5, JSON.stringify({ status: original.status, steady, frames: file.frames, gaps: file.gaps.slice(0, 5) }), ); check( 'its picture is the video at every frame: the colours turn exactly every 30 frames', colourOrder.includes('RGBW') && turned.slice(1).every((at, i) => at - turned[i] === 30), JSON.stringify(file.turns), ); check( 'and its sound is in step: the video heard from its first picture, the blip as green begins', file.toneBefore !== null && file.toneBefore < 0.005 && file.toneDuring > 0.02 && file.blipAt !== null && Math.abs(file.blipAt) <= 20 && steadyLines.some((text) => text.endsWith('with sound, not in real time')) && steadyLines.some((text) => text.includes('x real time, and has every frame')), JSON.stringify({ toneBefore: file.toneBefore, toneDuring: file.toneDuring, blipAtMs: file.blipAt, steadyLines, }), ); // 68k. Behind another tab Chrome draws no frames, and the run stood still and the recording with // it. While it records, a worker's beat ticks the page instead: not in real time the file has // every frame, the video's picture at each and the blip in step; in real time the run goes on // at its pace and the recording with it. const tabs = 'http://127.0.0.1:9333/json'; const ownTab = (await t.cdp.send('Target.getTargetInfo')).targetInfo.targetId; /** Records with the tab behind another, or `flipEvery` ms in front and behind in turn. */ const recordBehind = async (name, realTime, flipEvery = 0) => { await mutate(` const g = __gm.state.project.assets.find((a) => a.id === 'cp68-director').graph; Object.assign(g.nodes.find((n) => n.id === 'cp68-record').params, { name: ${JSON.stringify(name)}, realTime: ${realTime} });`); const before = await json('__gm.recording()'); const take = `__gm.recording().takes[${before.takes.length}]`; await evaluate(`(() => { window.__cp68frames = 0; window.__cp68count = true; const count = () => { if (!window.__cp68count) return; window.__cp68frames++; requestAnimationFrame(count); }; requestAnimationFrame(count); })()`); await play(); const other = await (await fetch(`${tabs}/new?about:blank`, { method: 'PUT' })).json(); const seen = { hidden: false, stillHidden: null, framesBehind: null, error: '' }; try { await fetch(`${tabs}/activate/${other.id}`); await waitUntil('document.hidden', { label: 'the tab went behind another' }); seen.hidden = true; seen.flips = 0; for (let front = false; flipEvery > 0 && seen.flips < 200; seen.flips++) { if (['done', 'failed'].includes(await json(`${take}?.state`))) break; await new Promise((resolve) => setTimeout(resolve, flipEvery)); front = !front; await fetch(`${tabs}/activate/${front ? ownTab : other.id}`); } const framesAt = await json('window.__cp68frames'); await waitUntil(`${take}?.state === 'done' || ${take}?.state === 'failed'`, { timeout: 60_000, label: `"${name}" is saved behind another tab`, }); seen.framesBehind = (await json('window.__cp68frames')) - framesAt; seen.stillHidden = await json('document.hidden'); } catch (error) { seen.error = String(error?.message ?? error); } finally { await fetch(`${tabs}/activate/${ownTab}`); await fetch(`${tabs}/close/${other.id}`); } await waitUntil('!document.hidden', { label: 'the tab is in front again' }); await evaluate('window.__cp68count = false; 0'); await stop(); const made = await json(take); if (made?.state !== 'done') return { made, seen, beats: 0, file: null }; await waitUntil( `__gm.recording().uploads.length > ${before.uploads.length} && ['kept', 'failed'].includes(__gm.recording().uploads.at(-1).state)`, { timeout: 30_000, label: 'the recording is uploaded' }, ); const after = await json('__gm.recording()'); return { made, seen, beats: after.beats - before.beats, file: await readRecording(after.uploads.at(-1)) }; }; const behind = await recordBehind('cp68 behind', false); const behindFile = behind.file; check( 'behind another tab, where the page draws no frame, a recording not in real time still has every frame', behind.seen.hidden && behind.seen.stillHidden === true && behind.seen.framesBehind <= 2 && behind.beats > 100 && behind.made?.failure === null && behindFile?.ok && behindFile.frames >= 120 && behindFile.gaps.length === 0, JSON.stringify({ seen: behind.seen, beats: behind.beats, made: behind.made, frames: behindFile?.frames, gaps: behindFile?.gaps.slice(0, 5), }), ); check( 'and its picture and sound are those in front: colours every 30 frames, the blip as green begins', Boolean(behindFile) && behindFile.colourOrder.includes('RGBW') && behindFile.turned.slice(1).every((at, i) => at - behindFile.turned[i] === 30) && behindFile.blipAt !== null && Math.abs(behindFile.blipAt) <= 20, JSON.stringify({ turns: behindFile?.turns, blipAtMs: behindFile?.blipAt, toneDuring: behindFile?.toneDuring, }), ); const live = await recordBehind('cp68 behind live', true); check( 'and in real time the run goes on behind another tab, and the recording with it', live.seen.hidden && live.seen.stillHidden === true && live.seen.framesBehind <= 2 && live.beats > 60 && live.made?.failure === null && live.file?.ok && // Half its frames at least: in front of the others it has about 60 % here, the encoder in WSL // being slow; standing still behind them it had none. live.file.frames >= live.made.durationMs * 0.03, JSON.stringify({ seen: live.seen, beats: live.beats, made: live.made, frames: live.file?.frames }), ); // 68l. A video item that begins behind another tab: Chrome hands over no frame there, and its // player, opened there, waited for one until the tab was in front again. Every frame meanwhile // waited 3 s and was taken without the video, and without its sound. And switched between the // tabs while it records: a picture learnt too long from a step across the time behind held // the video for several frames, its colours turned late. const exact = (file) => JSON.stringify(file?.turns) === JSON.stringify(['0R', '30G', '60B', '90W']); // A frame late at most: a seek just after the tab came in front once landed with the picture before // (1 run in 5). Learnt too long, the picture held 2 to 6 frames late. const nearly = (file) => file?.turns.length === 4 && file.turns.every( (turn, i) => turn.endsWith('RGBW'[i]) && Math.abs(Number.parseInt(turn, 10) - 30 * i) <= 1, ); await mutate(` const c = __gm.state.project.assets.find((a) => a.id === 'cp68-cut'); const v = c.objects.find((o) => o.id === 'cp68-v'); c.objects.push({ ...structuredClone(v), id: 'cp68-v2', name: 'Video 68 b' }); c.items.find((i) => i.id === 'cp68-v').length = 108; c.items.push({ id: 'cp68-v2', track: 'cp68-front', start: 108, length: 12, offset: 108 });`); const lateCut = await recordBehind('cp68 cut behind', false); check( 'a video item that begins behind another tab is in the recording from its first frame on', lateCut.seen.stillHidden === true && lateCut.made?.failure === null && lateCut.file?.ok && lateCut.file.frames >= 120 && lateCut.file.gaps.length === 0 && exact(lateCut.file) && lateCut.made.tookMs < 15_000, JSON.stringify({ seen: lateCut.seen, made: lateCut.made, frames: lateCut.file?.frames, turns: lateCut.file?.turns, }), ); const flip = await recordBehind('cp68 flip', false, 350); check( 'switched between the tabs every 350 ms, its colours turn every 30 frames, a frame late at most', flip.seen.flips >= 4 && flip.made?.failure === null && flip.file?.ok && flip.file.gaps.length === 0 && nearly(flip.file) && flip.file.blipAt !== null && Math.abs(flip.file.blipAt) <= 20, JSON.stringify({ seen: flip.seen, made: flip.made, turns: flip.file?.turns, blipAtMs: flip.file?.blipAt, }), ); await mutate(` const p = __gm.state.project; p.assets = p.assets.filter((a) => !a.id.startsWith('cp68-') && a.id !== ${JSON.stringify(madeId)}); const scene = __gm.state.activeScene; scene.objects = scene.objects.filter((o) => !o.id.startsWith('cp68-'));`); }