Keep a video item in a recording when the tab goes back and forth

A recording made while switching between tabs had stretches without
the video item and without its sound, and in between the video stood
for several frames. Two things went wrong behind another tab.

- A player opened there got no first picture. It waited for
  requestVideoFrameCallback, which Chrome never calls behind a tab.
  A recording not in real time then waited 3 s per frame and took it
  without the video. Now the player takes its first picture as soon
  as it has data while the tab is hidden.
- The span of a picture was learnt from steps that skipped pictures.
  After a time behind the tab it learnt 0.133 s instead of 1/30 s,
  left out seeks within that span and never corrected it. Now it
  learns only from steps where the time sought moved at most half a
  step, so no picture can lie between them.
- Behind the tab a seek forgets which picture shows but keeps the
  span. A landing without a frame counts as the same picture only
  once the tab has been in front for 500 ms.

Found in the user's recording 7ea9146b: 5363 frames, all 1/60 s
apart, with the video missing at 10.63 s (8 frames), 43.17 s
(5 frames) and 46.93 s (28 frames), and the sound gone with it.

Tests:
- Unit tests/viewport: 377 green, 4 new or rebuilt. With the old
  rule and the first picture only by requestVideoFrameCallback both
  new tests go red.
- e2e 68 on melektaus-atx: ALL PASSED, 21 checks. New 68l: a video
  item that begins behind another tab has its colours exactly, and
  a tab switched every 350 ms gives colour changes at 0/30/60/90.
- One run in five with switches had a change one frame late, so the
  check allows one frame either way.
- Gate on melektaus-atx: green, 311 files, 4824 tests.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

README.md

@@ -47,7 +47,7 @@ npm run build        # writes dist/ and regenerates index.html; the folder is wh
 The browser suite and the three example games need a machine with a GPU:
 
 ```bash
-node tools/seed-e2e.mjs && node tools/e2e.mjs      # the editor, 2086 checks (GM_STEPS=20 for one step)
+node tools/seed-e2e.mjs && node tools/e2e.mjs      # the editor, 2088 checks (GM_STEPS=20 for one step)
                                                    #   every tool signs in first – see the Accounts page
 node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs
 node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs

src/viewport/VideoLibrary.ts

@@ -23,12 +23,15 @@ import {
   type VideoPlay,
 } from '../core/model/video';
 import { videoDrivenOf, videoLevelOf, videoTimeOf } from '../core/properties/videos';
+import { every } from './backgroundTimers';
 import { FileLibrary } from './FileLibrary';
 import { soundLibrary } from './SoundLibrary';
 import type { HeardVideo } from './SteadySound';
 import {
   followVideo,
   PICTURE_NUDGE,
+  pictureKept,
+  pictureLost,
   pictureShown,
   readSpeed,
   type ShownPicture,
@@ -53,6 +56,11 @@ const IDLE_MS = 2000;
  * itself otherwise, heard, until it is let go. A free one (#420) plays on from there once asked again.
  */
 const HUSH_MS = 100;
+/** How often a player opened behind another tab looks for its first picture, see `openVideo`. */
+const HIDDEN_PICTURE_MS = 20;
+/** `HTMLMediaElement`'s: let go (see `drop`), and with the picture of where it stands. */
+const NETWORK_EMPTY = 0;
+const HAVE_CURRENT_DATA = 2;
 /** Seconds a driven video's level takes to get most of the way to a new one, see `hear`. */
 const LEVEL_GLIDE = 0.015;
 
@@ -390,9 +398,13 @@ export class VideoLibrary {
         player.element,
         drive,
         () => player.gone,
-        (shown) => {
+        (shown, inFront) => {
           player.texture?.source.update();
-          if (shown !== null && player.element.paused) drive.shown = pictureShown(drive.shown, shown);
+          const { element } = player;
+          if (!element.paused) return;
+          if (shown !== null) drive.shown = pictureShown(drive.shown, shown, element.currentTime);
+          else
+            drive.shown = inFront ? pictureKept(drive.shown, element.currentTime) : pictureLost(drive.shown);
         },
       );
     else {
@@ -494,9 +506,21 @@ export function openVideo(url: string, at: number, onPicture: (texture: Texture)
   element.addEventListener(
     'loadeddata',
     () => {
-      element.requestVideoFrameCallback(() =>
-        onPicture(new Texture({ source: new VideoSource({ resource: element, autoPlay: false }) })),
-      );
+      let made = false;
+      const make = () => {
+        if (made) return;
+        made = true;
+        stopLooking();
+        onPicture(new Texture({ source: new VideoSource({ resource: element, autoPlay: false }) }));
+      };
+      // Behind another tab Chrome hands over no frame, and a player opened there stayed empty until
+      // the tab was in front again: a recording not in real time waited 3 s for each of its frames
+      // and took them without it (#423). Its first picture is there all the same once it has data.
+      const stopLooking = every(HIDDEN_PICTURE_MS, () => {
+        if (element.networkState === NETWORK_EMPTY) stopLooking();
+        else if (document.hidden && element.readyState >= HAVE_CURRENT_DATA) make();
+      });
+      element.requestVideoFrameCallback(make);
       void element.play().catch(() => {});
     },
     { once: true },

src/viewport/videoFollow.ts

@@ -98,22 +98,49 @@ export function followVideo({ target, speed, current, end }: FollowState): Follo
 }
 
 /**
- * The picture a held video shows (#423): where it begins in the file, and how long a picture of
- * the video lasts, as learnt from the pictures shown so far; null while not known.
+ * The picture a held video shows (#423): where it begins in the file, null while not known, and the
+ * latest time sought that it is known to show. `span` is how long a picture of the video lasts, as
+ * learnt from the pictures shown so far; null while not known.
  */
 export interface ShownPicture {
-  at: number;
+  at: number | null;
+  reach: number;
   span: number | null;
 }
 
 /**
- * The picture shown once Chrome hands over the one at `at`: the shortest step between two
- * pictures seen is how long one lasts. A seek that steps one picture on teaches it at once.
+ * The picture shown once Chrome hands over the one at `at` for a seek to `sought`. A step between
+ * two pictures teaches how long one lasts only when the time sought moved at most half that step
+ * since the picture before was known to show: then no picture can lie between them. A span learnt
+ * from a longer step was too long, so seeks were left out within it and no shorter step came to
+ * correct it: the video stood for several frames of a recording. Such steps came after a time
+ * behind another tab, a jump, or a seek that stepped two pictures on.
  */
-export function pictureShown(before: ShownPicture | null, at: number): ShownPicture {
-  const step = before ? Math.abs(at - before.at) : 0;
+export function pictureShown(before: ShownPicture | null, at: number, sought: number): ShownPicture {
   const span = before?.span ?? null;
-  return { at, span: step > 1e-6 && (span === null || step < span) ? step : span };
+  if (!before || before.at === null) return { at, reach: sought, span };
+  const step = Math.abs(at - before.at);
+  if (step <= 1e-6) return { at: before.at, reach: Math.max(before.reach, sought), span };
+  // A tenth of a millisecond to spare: Chrome hands over a picture's time in microseconds.
+  const learns = Math.abs(sought - before.reach) <= step / 2 + 1e-4 && (span === null || step < span);
+  return { at, reach: sought, span: learns ? step : span };
+}
+
+/**
+ * A seek to `sought` that brought no frame though the page is in front: Chrome hands over none for
+ * a seek within the picture shown, so it still shows.
+ */
+export function pictureKept(before: ShownPicture | null, sought: number): ShownPicture | null {
+  if (!before || before.at === null) return before;
+  return { ...before, reach: Math.max(before.reach, sought) };
+}
+
+/**
+ * A seek behind another tab, where Chrome hands over no frame at all: which picture shows is not
+ * known, how long one lasts still is.
+ */
+export function pictureLost(before: ShownPicture | null): ShownPicture | null {
+  return before && { at: null, reach: before.reach, span: before.span };
 }
 
 /**
@@ -122,7 +149,7 @@ export function pictureShown(before: ShownPicture | null, at: number): ShownPict
  * Where the next picture begins belongs to that one, see `PICTURE_NUDGE`.
  */
 export function showsPictureOf(shown: ShownPicture | null, target: number): boolean {
-  if (!shown || shown.span === null) return false;
+  if (!shown || shown.at === null || shown.span === null) return false;
   return target >= shown.at - 1e-6 && target < shown.at + shown.span - 1e-6;
 }
 

src/viewport/videoSeek.ts

@@ -13,6 +13,18 @@ const SEEK_SLACK = 1 / 120;
 const LAND_MS = 150;
 /** The same behind another tab, where Chrome hands over no frame at all and the picture is there at `seeked`. */
 const HIDDEN_LAND_MS = 10;
+/**
+ * How long the page has to be in front again before a seek with no frame counts as one within the
+ * picture shown: Chrome may take a moment to hand over frames again.
+ */
+const FRONT_SETTLE_MS = 500;
+
+/** When the page came in front last, by `performance.now()`. */
+let frontSince = 0;
+if (typeof document !== 'undefined' && typeof document.addEventListener === 'function')
+  document.addEventListener('visibilitychange', () => {
+    if (!document.hidden) frontSince = performance.now();
+  });
 
 /** One element's seeks, see `seekVideo` and `chaseSeeks`. */
 export interface SeekChase {
@@ -42,13 +54,15 @@ export function seekVideo(element: HTMLVideoElement, chase: SeekChase, time: num
  * loads a paused video's picture only then, kept the one before it for a frame. `landed` is where
  * the caller loads it again, told the time of that picture in the file. Chrome hands over no frame
  * for some seeks, though – one within the picture shown, for one – and a player waiting for one
- * stayed empty until the next jump; after `LAND_MS` it lands without.
+ * stayed empty until the next jump; after `LAND_MS` it lands without. `inFront` tells whether the
+ * page was in front all the while, and for a moment before: behind another tab Chrome hands over
+ * no frame for any seek.
  */
 export function chaseSeeks(
   element: HTMLVideoElement,
   chase: SeekChase,
   gone: () => boolean,
-  landed: (shown: number | null) => void,
+  landed: (shown: number | null, inFront: boolean) => void,
 ): void {
   let round = 0;
   // A held video may hand over the picture of a seek before `seeked`: a callback asked for only
@@ -73,15 +87,21 @@ export function chaseSeeks(
     }
     if (!chase.landing) return;
     const now = ++round;
+    const behind = isHidden();
+    const settled = !behind && performance.now() - frontSince >= FRONT_SETTLE_MS;
     const land = (shown: number | null) => {
       if (gone() || now !== round || element.seeking || !chase.landing) return;
-      landed(shown);
+      landed(shown, settled && !isHidden());
       chase.landing = false;
     };
     if (early !== null) return land(early);
     element.requestVideoFrameCallback((_now, frame) => land(frame.mediaTime));
     // A worker's timer: behind another tab Chrome holds the page's own to once a second, and a
     // recording not in real time waits for every landing.
-    after(typeof document !== 'undefined' && document.hidden ? HIDDEN_LAND_MS : LAND_MS, () => land(null));
+    after(behind ? HIDDEN_LAND_MS : LAND_MS, () => land(null));
   });
 }
+
+function isHidden(): boolean {
+  return typeof document !== 'undefined' && document.hidden;
+}

tests/viewport/video-follow.test.ts

@@ -2,6 +2,8 @@ import { describe, expect, it } from 'vitest';
 import {
   followVideo,
   PICTURE_NUDGE,
+  pictureKept,
+  pictureLost,
   pictureShown,
   readSpeed,
   SEEK_DRIFT,
@@ -133,32 +135,62 @@ describe('following the asked time', () => {
 
 describe('the picture a held video shows (#423)', () => {
   it('learns how long a picture lasts from the pictures Chrome hands over', () => {
-    let shown = pictureShown(null, 0);
-    expect(shown).toEqual({ at: 0, span: null });
-    shown = pictureShown(shown, 0);
+    // A video at 30 fps sought at 60: every other seek stays within the picture shown.
+    let shown = pictureShown(null, 0, 0.0005);
+    expect(shown).toEqual({ at: 0, reach: 0.0005, span: null });
+    shown = pictureShown(shown, 0, 1 / 60);
+    expect(shown).toEqual({ at: 0, reach: 1 / 60, span: null });
+    shown = pictureShown(shown, 1 / 30, 2 / 60);
+    expect(shown.span).toBeCloseTo(1 / 30, 12);
+    // A jump back or far on teaches nothing.
+    shown = pictureShown(shown, 1, 1.0005);
+    expect(shown).toEqual({ at: 1, reach: 1.0005, span: shown.span });
+    expect(shown.span).toBeCloseTo(1 / 30, 12);
+  });
+
+  it('learns nothing from a step two pictures long, which would hold the video for good', () => {
+    // Sought one frame of 60 on, but from a picture known only at its start: two pictures on came.
+    let shown = pictureShown(null, 0, 0.0005);
+    shown = pictureShown(shown, 2 / 30, 2 / 30 + 0.0005);
     expect(shown.span).toBeNull();
-    shown = pictureShown(shown, 2 / 30);
-    expect(shown.span).toBeCloseTo(2 / 30, 12);
-    shown = pictureShown(shown, 3 / 30);
-    expect(shown.span).toBeCloseTo(1 / 30, 12);
-    // A jump back or far on teaches nothing shorter.
-    shown = pictureShown(shown, 1);
-    expect(shown).toEqual({ at: 1, span: shown.span });
+    // Learnt so, every seek within the two pictures was left out and none came to correct it.
+    expect(showsPictureOf(shown, 3 / 30)).toBe(false);
+    // A seek within the picture brings no frame in front of the others: it still shows, up to there.
+    shown = pictureKept(shown, 2 / 30 + 1 / 60 + 0.0005)!;
+    shown = pictureShown(shown, 3 / 30, 3 / 30 + 0.0005);
     expect(shown.span).toBeCloseTo(1 / 30, 12);
   });
 
+  it('forgets the picture shown behind another tab, and keeps how long one lasts', () => {
+    const learnt = { at: 0.5, reach: 0.51, span: 1 / 30 };
+    const lost = pictureLost(learnt)!;
+    expect(lost.span).toBe(1 / 30);
+    expect(showsPictureOf(lost, 0.51)).toBe(false);
+    expect(pictureKept(lost, 0.6)).toEqual(lost);
+    // The first picture handed over in front again is where it stands; the step to it teaches nothing.
+    const back = pictureShown(lost, 1.5, 1.5005);
+    expect(back).toEqual({ at: 1.5, reach: 1.5005, span: 1 / 30 });
+    // Known only behind a tab, the span is not learnt from a picture far from the one before.
+    expect(pictureShown(pictureLost({ at: 0.5, reach: 0.5, span: null }), 0.9, 0.9)).toEqual({
+      at: 0.9,
+      reach: 0.9,
+      span: null,
+    });
+    expect(pictureLost(null)).toBeNull();
+  });
+
   it('knows a seek within the picture shown brings no other; where the next begins is the next', () => {
     // As measured: frames of a video at 30 fps sought at 60.
-    const shown = { at: 1 / 30, span: 1 / 30 };
+    const shown = { at: 1 / 30, reach: 1 / 30, span: 1 / 30 };
     expect(showsPictureOf(shown, 1 / 30)).toBe(true);
     expect(showsPictureOf(shown, 1.5 / 30)).toBe(true);
     expect(showsPictureOf(shown, 2 / 30 - 0.0005)).toBe(true);
     expect(showsPictureOf(shown, 4 / 60)).toBe(false);
     expect(showsPictureOf(shown, 5 / 60)).toBe(false);
     expect(showsPictureOf(shown, 0.02)).toBe(false);
-    expect(showsPictureOf({ at: 1 / 30, span: null }, 1.5 / 30)).toBe(false);
+    expect(showsPictureOf({ at: 1 / 30, reach: 1 / 30, span: null }, 1.5 / 30)).toBe(false);
     expect(showsPictureOf(null, 0)).toBe(false);
     // A seek half a millisecond on stays within its picture, the shortest a video has (60 fps).
-    expect(showsPictureOf({ at: 0.5, span: 1 / 60 }, 0.5 + PICTURE_NUDGE)).toBe(true);
+    expect(showsPictureOf({ at: 0.5, reach: 0.5, span: 1 / 60 }, 0.5 + PICTURE_NUDGE)).toBe(true);
   });
 });

tests/viewport/video-library.test.ts

@@ -213,6 +213,44 @@ describe('VideoLibrary driven players (#404)', () => {
     expect(videos.showsAsked(key)).toBe(true);
   });
 
+  it('gives a player opened behind another tab its first picture, with no frame handed over (#423)', async () => {
+    vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] });
+    const page = { hidden: true };
+    const events = new Map<string, () => void>();
+    const element = {
+      ...fakeVideo(),
+      seeking: false,
+      ended: false,
+      duration: 2,
+      playbackRate: 1,
+      readyState: 4,
+      networkState: 1,
+      addEventListener: (type: string, listener: () => void) => events.set(type, listener),
+      // Behind another tab Chrome calls none of these.
+      requestVideoFrameCallback: () => 0,
+    };
+    vi.stubGlobal('document', {
+      get hidden() {
+        return page.hidden;
+      },
+      addEventListener: () => {},
+      createElement: () => element,
+    });
+    const videos = await freshLibrary();
+    const key = videos.wear(REF, { videoClock: 'driven', videoTime: 0.5 }, 'view1:a', 'game').file;
+    events.get('loadeddata')!();
+    vi.advanceTimersByTime(100);
+    const behind = videos.hasPicture(key);
+    // In front Chrome hands over the frame, and the player waits for it.
+    page.hidden = false;
+    const front = videos.wear(REF, { videoClock: 'driven', videoTime: 0.5 }, 'view1:b', 'game').file;
+    events.get('loadeddata')!();
+    vi.advanceTimersByTime(100);
+    vi.useRealTimers();
+    expect(behind).toBe(true);
+    expect(videos.hasPicture(front)).toBe(false);
+  });
+
   it('holds a driven player quiet as soon as its wearer is gone, as an ended composition item', async () => {
     vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval', 'performance'] });
     const events = new Map<string, () => void>();

tools/e2e/68-composition.mjs

@@ -16,7 +16,9 @@ import { MAKE_OBJECT } from './lib/scaffold.mjs';
  * ends, its video stops and goes quiet with it. A sound item is planned once the clock runs, timed
  * to the cut. Recorded not in real time from a slowed editor, the file still has every frame, the
  * video's picture at each, and its sound and the sound item's in step. Behind another tab, where the
- * page draws no frames, a recording goes on all the same, not in real time and in real time.
+ * page draws no frames, a recording goes on all the same, not in real time and in real time. A video
+ * item that begins there is in it from its first frame, and switched between the tabs while it
+ * records, its video still turns in step.
  *
  * Everything it adds carries a `cp68` id and goes again at the end.
  */
@@ -647,7 +649,8 @@ export async function composition(t) {
   //      at its pace and the recording with it.
   const tabs = 'http://127.0.0.1:9333/json';
   const ownTab = (await t.cdp.send('Target.getTargetInfo')).targetInfo.targetId;
-  const recordBehind = async (name, realTime) => {
+  /** Records with the tab behind another, or `flipEvery` ms in front and behind in turn. */
+  const recordBehind = async (name, realTime, flipEvery = 0) => {
     await mutate(`
       const g = __gm.state.project.assets.find((a) => a.id === 'cp68-director').graph;
       Object.assign(g.nodes.find((n) => n.id === 'cp68-record').params, { name: ${JSON.stringify(name)}, realTime: ${realTime} });`);
@@ -670,6 +673,13 @@ export async function composition(t) {
       await fetch(`${tabs}/activate/${other.id}`);
       await waitUntil('document.hidden', { label: 'the tab went behind another' });
       seen.hidden = true;
+      seen.flips = 0;
+      for (let front = false; flipEvery > 0 && seen.flips < 200; seen.flips++) {
+        if (['done', 'failed'].includes(await json(`${take}?.state`))) break;
+        await new Promise((resolve) => setTimeout(resolve, flipEvery));
+        front = !front;
+        await fetch(`${tabs}/activate/${front ? ownTab : other.id}`);
+      }
       const framesAt = await json('window.__cp68frames');
       await waitUntil(`${take}?.state === 'done' || ${take}?.state === 'failed'`, {
         timeout: 60_000,
@@ -743,6 +753,60 @@ export async function composition(t) {
     JSON.stringify({ seen: live.seen, beats: live.beats, made: live.made, frames: live.file?.frames }),
   );
 
+  // 68l. A video item that begins behind another tab: Chrome hands over no frame there, and its
+  //      player, opened there, waited for one until the tab was in front again. Every frame meanwhile
+  //      waited 3 s and was taken without the video, and without its sound. And switched between the
+  //      tabs while it records: a picture learnt too long from a step across the time behind held
+  //      the video for several frames, its colours turned late.
+  const exact = (file) => JSON.stringify(file?.turns) === JSON.stringify(['0R', '30G', '60B', '90W']);
+  // A frame late at most: a seek just after the tab came in front once landed with the picture before
+  // (1 run in 5). Learnt too long, the picture held 2 to 6 frames late.
+  const nearly = (file) =>
+    file?.turns.length === 4 &&
+    file.turns.every(
+      (turn, i) => turn.endsWith('RGBW'[i]) && Math.abs(Number.parseInt(turn, 10) - 30 * i) <= 1,
+    );
+  await mutate(`
+    const c = __gm.state.project.assets.find((a) => a.id === 'cp68-cut');
+    const v = c.objects.find((o) => o.id === 'cp68-v');
+    c.objects.push({ ...structuredClone(v), id: 'cp68-v2', name: 'Video 68 b' });
+    c.items.find((i) => i.id === 'cp68-v').length = 108;
+    c.items.push({ id: 'cp68-v2', track: 'cp68-front', start: 108, length: 12, offset: 108 });`);
+  const lateCut = await recordBehind('cp68 cut behind', false);
+  check(
+    'a video item that begins behind another tab is in the recording from its first frame on',
+    lateCut.seen.stillHidden === true &&
+      lateCut.made?.failure === null &&
+      lateCut.file?.ok &&
+      lateCut.file.frames >= 120 &&
+      lateCut.file.gaps.length === 0 &&
+      exact(lateCut.file) &&
+      lateCut.made.tookMs < 15_000,
+    JSON.stringify({
+      seen: lateCut.seen,
+      made: lateCut.made,
+      frames: lateCut.file?.frames,
+      turns: lateCut.file?.turns,
+    }),
+  );
+  const flip = await recordBehind('cp68 flip', false, 350);
+  check(
+    'switched between the tabs every 350 ms, its colours turn every 30 frames, a frame late at most',
+    flip.seen.flips >= 4 &&
+      flip.made?.failure === null &&
+      flip.file?.ok &&
+      flip.file.gaps.length === 0 &&
+      nearly(flip.file) &&
+      flip.file.blipAt !== null &&
+      Math.abs(flip.file.blipAt) <= 20,
+    JSON.stringify({
+      seen: flip.seen,
+      made: flip.made,
+      turns: flip.file?.turns,
+      blipAtMs: flip.file?.blipAt,
+    }),
+  );
+
   await mutate(`
     const p = __gm.state.project;
     p.assets = p.assets.filter((a) => !a.id.startsWith('cp68-') && a.id !== ${JSON.stringify(madeId)});