diff --git a/README.md b/README.md index 709d990d..f077ff63 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh The browser suite and the three example games need a machine with a GPU: ```bash -node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2067 checks (GM_STEPS=20 for one step) +node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2068 checks (GM_STEPS=20 for one step) # every tool signs in first – see the Accounts page node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs diff --git a/src/core/recording/avcColour.ts b/src/core/recording/avcColour.ts new file mode 100644 index 00000000..c969f4cb --- /dev/null +++ b/src/core/recording/avcColour.ts @@ -0,0 +1,279 @@ +/** + * The colour of an H.264 stream, written where a player reads it: in the `video_signal_type` of its + * sequence parameter set (SPS). Chrome's encoder leaves that out for some inputs – a frame of RGBX + * bytes comes out in the full range, and the SPS says nothing – and a player that reads only the + * stream (VLC 3.0.8) takes the broadcast range then: the picture comes out darker, its colours off. + * The Matroska `Colour` says it too, but not every player reads it. + * + * Only the SPS changes; the frames stay as they were. With `avc: {format: 'avc'}` the SPS is in the + * encoder's `description` (the `avcC`) alone, so that is all there is to rewrite. + */ +import type { MatroskaColour } from './matroska'; + +/** What `video_format` says when the SPS had none: "unspecified". */ +const UNSPECIFIED_FORMAT = 5; +const SPS_TYPE = 7; +/** The profiles whose SPS carries the chroma format, bit depths and scaling lists (H.264 7.3.2.1.1). */ +const HIGH_PROFILES = new Set([100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134, 135]); + +/** + * The `avcC` with the colour in each of its SPS; the rest – the PPS and what follows them – is kept + * byte for byte. Throws on an `avcC` or SPS it cannot read. + */ +export function avcConfigWithColour(config: Uint8Array, colour: MatroskaColour): Uint8Array { + if (config.length < 7 || config[0] !== 1) throw new Error('not an avcC'); + const count = config[5]! & 0x1f; + const parts: Uint8Array[] = [config.subarray(0, 6)]; + let at = 6; + for (let i = 0; i < count; i++) { + const length = (config[at]! << 8) | config[at + 1]!; + const nal = config.subarray(at + 2, at + 2 + length); + if (nal.length !== length) throw new Error('avcC ends inside an SPS'); + const sps = spsWithColour(nal, colour); + parts.push(Uint8Array.of(sps.length >> 8, sps.length & 0xff), sps); + at += 2 + length; + } + parts.push(config.subarray(at)); + return concat(parts); +} + +/** The SPS of an `avcC`, as NAL units with their header byte. */ +export function avcConfigSps(config: Uint8Array): Uint8Array[] { + const out: Uint8Array[] = []; + let at = 6; + for (let i = 0; i < (config[5]! & 0x1f); i++) { + const length = (config[at]! << 8) | config[at + 1]!; + out.push(config.subarray(at + 2, at + 2 + length)); + at += 2 + length; + } + return out; +} + +/** + * One SPS (a NAL unit with its header byte) with this colour in its `video_signal_type`. A VUI it + * had keeps everything else; without one it gets one that says the colour and nothing more. + */ +export function spsWithColour(nal: Uint8Array, colour: MatroskaColour): Uint8Array { + if ((nal[0]! & 0x1f) !== SPS_TYPE) throw new Error('not an SPS'); + const rbsp = unescape(nal.subarray(1)); + const read = new BitReader(rbsp); + skipToVui(read); + const out = new BitWriter(); + const hasVui = read.bit() === 1; + let videoFormat = UNSPECIFIED_FORMAT; + let rest: { from: number; to: number } | null = null; + if (hasVui) { + if (read.bit()) { + if (read.bits(8) === 255) read.bits(32); + } + if (read.bit()) read.bit(); + const signalAt = read.at; + out.copy(rbsp, 0, signalAt); + if (read.bit()) { + videoFormat = read.bits(3); + read.bit(); + if (read.bit()) read.bits(24); + } + rest = { from: read.at, to: stopBitOf(rbsp) }; + } else { + out.copy(rbsp, 0, read.at - 1); + out.bit(1); + // No aspect ratio, no overscan. + out.bits(0, 2); + } + out.bit(1); + out.bits(videoFormat, 3); + out.bit(colour.fullRange ? 1 : 0); + out.bit(1); + out.bits(colour.primaries, 8); + out.bits(colour.transfer, 8); + out.bits(colour.matrix, 8); + if (rest) out.copy(rbsp, rest.from, rest.to); + // No chroma location, timing, HRD, picture structure or bitstream restriction. + else out.bits(0, 6); + out.bit(1); + return concat([nal.subarray(0, 1), escape(out.finish())]); +} + +/** What the `video_signal_type` of an SPS says; null where it has none. */ +export function spsColour(nal: Uint8Array): MatroskaColour | null { + const read = new BitReader(unescape(nal.subarray(1))); + skipToVui(read); + if (!read.bit()) return null; + if (read.bit()) { + if (read.bits(8) === 255) read.bits(32); + } + if (read.bit()) read.bit(); + if (!read.bit()) return null; + read.bits(3); + const fullRange = read.bit() === 1; + if (!read.bit()) return { matrix: 2, transfer: 2, primaries: 2, fullRange }; + const primaries = read.bits(8); + const transfer = read.bits(8); + const matrix = read.bits(8); + return { matrix, transfer, primaries, fullRange }; +} + +/** Reads an SPS up to its `vui_parameters_present_flag`. */ +function skipToVui(read: BitReader): void { + const profile = read.bits(8); + read.bits(16); + read.ue(); + if (HIGH_PROFILES.has(profile)) { + const chroma = read.ue(); + if (chroma === 3) read.bit(); + read.ue(); + read.ue(); + read.bit(); + if (read.bit()) + for (let i = 0; i < (chroma === 3 ? 12 : 8); i++) + if (read.bit()) skipScalingList(read, i < 6 ? 16 : 64); + } + read.ue(); + const pocType = read.ue(); + if (pocType === 0) read.ue(); + else if (pocType === 1) { + read.bit(); + read.se(); + read.se(); + const cycle = read.ue(); + for (let i = 0; i < cycle; i++) read.se(); + } + read.ue(); + read.bit(); + read.ue(); + read.ue(); + if (!read.bit()) read.bit(); + read.bit(); + if (read.bit()) for (let i = 0; i < 4; i++) read.ue(); +} + +function skipScalingList(read: BitReader, size: number): void { + let last = 8; + let next = 8; + for (let j = 0; j < size; j++) { + if (next !== 0) next = (last + read.se() + 256) % 256; + last = next === 0 ? last : next; + } +} + +/** The bit the RBSP ends its data with, the last one set: what follows is padding. */ +function stopBitOf(rbsp: Uint8Array): number { + for (let byte = rbsp.length - 1; byte >= 0; byte--) { + const value = rbsp[byte]!; + if (value === 0) continue; + let bit = 7; + while (!(value & (1 << (7 - bit)))) bit--; + return byte * 8 + bit; + } + throw new Error('SPS without its stop bit'); +} + +/** The RBSP of a NAL unit's payload: each `00 00 03` loses its 03. */ +function unescape(payload: Uint8Array): Uint8Array { + const out: number[] = []; + let zeros = 0; + for (const value of payload) { + if (zeros >= 2 && value === 3) { + zeros = 0; + continue; + } + out.push(value); + zeros = value === 0 ? zeros + 1 : 0; + } + return Uint8Array.from(out); +} + +/** The payload of an RBSP: a 03 after two zeros wherever the next byte could be mistaken for a start. */ +function escape(rbsp: Uint8Array): Uint8Array { + const out: number[] = []; + let zeros = 0; + for (const value of rbsp) { + if (zeros >= 2 && value <= 3) { + out.push(3); + zeros = 0; + } + out.push(value); + zeros = value === 0 ? zeros + 1 : 0; + } + return Uint8Array.from(out); +} + +class BitReader { + at = 0; + private readonly bytes: Uint8Array; + + constructor(bytes: Uint8Array) { + this.bytes = bytes; + } + + bit(): number { + const byte = this.bytes[this.at >> 3]; + if (byte === undefined) throw new Error('SPS ends early'); + const value = (byte >> (7 - (this.at & 7))) & 1; + this.at++; + return value; + } + + bits(count: number): number { + let value = 0; + for (let i = 0; i < count; i++) value = value * 2 + this.bit(); + return value; + } + + /** Exp-Golomb, unsigned. */ + ue(): number { + let zeros = 0; + while (this.bit() === 0) { + zeros++; + if (zeros > 31) throw new Error('SPS has a number out of range'); + } + return 2 ** zeros - 1 + this.bits(zeros); + } + + /** Exp-Golomb, signed. */ + se(): number { + const code = this.ue(); + return code % 2 ? (code + 1) / 2 : -code / 2; + } +} + +class BitWriter { + private readonly out: number[] = []; + private current = 0; + private filled = 0; + + bit(value: number): void { + this.current = (this.current << 1) | (value & 1); + if (++this.filled === 8) { + this.out.push(this.current); + this.current = 0; + this.filled = 0; + } + } + + bits(value: number, count: number): void { + for (let i = count - 1; i >= 0; i--) this.bit(Math.floor(value / 2 ** i) % 2); + } + + /** The bits `from` up to `to` of these bytes, as they are. */ + copy(bytes: Uint8Array, from: number, to: number): void { + for (let at = from; at < to; at++) this.bit((bytes[at >> 3]! >> (7 - (at & 7))) & 1); + } + + /** The bytes, the last one filled up with zeros (after the stop bit the caller wrote). */ + finish(): Uint8Array { + if (this.filled > 0) this.bits(0, 8 - this.filled); + return Uint8Array.from(this.out); + } +} + +function concat(parts: readonly Uint8Array[]): Uint8Array { + const out = new Uint8Array(parts.reduce((sum, part) => sum + part.length, 0)); + let at = 0; + for (const part of parts) { + out.set(part, at); + at += part.length; + } + return out; +} diff --git a/src/core/recording/recording.ts b/src/core/recording/recording.ts index 0afcffc9..cb4b4c89 100644 --- a/src/core/recording/recording.ts +++ b/src/core/recording/recording.ts @@ -104,6 +104,41 @@ function even(value: number): number { return Math.max(2, Math.round(value / 2) * 2); } +/** + * Where the planes of an I420 frame of `w` × `h` (both even) lie in a picture read back from the + * GPU, 4 bytes a texel: the luma rows first, then Cb, then Cr. A row of luma is `stride` bytes, a + * multiple of 8, so a texel never holds bytes of two chroma rows; a texel row holds two chroma + * rows, so each chroma plane takes `chromaRows` texel rows, the last one half used when `h / 2` is + * odd. `layout` is the `VideoFrame`'s, `texels` the size of the picture to draw the planes into. + */ +export function i420Layout( + w: number, + h: number, +): { + stride: number; + chromaRows: number; + texels: { w: number; h: number }; + bytes: number; + layout: { offset: number; stride: number }[]; +} { + const stride = Math.ceil(w / 8) * 8; + const chromaRows = Math.ceil(h / 4); + const rows = h + 2 * chromaRows; + const cb = stride * h; + const cr = cb + stride * chromaRows; + return { + stride, + chromaRows, + texels: { w: stride / 4, h: rows }, + bytes: stride * rows, + layout: [ + { offset: 0, stride }, + { offset: cb, stride: stride / 2 }, + { offset: cr, stride: stride / 2 }, + ], + }; +} + /** * The size of an editor recording of a canvas in device pixels: all of it, an odd row or column * cut off, since the encoders want both sides even. diff --git a/src/viewport/GameRecorder.ts b/src/viewport/GameRecorder.ts index 10472bd1..fefcefe9 100644 --- a/src/viewport/GameRecorder.ts +++ b/src/viewport/GameRecorder.ts @@ -111,9 +111,20 @@ export class GameRecorder implements RecordingSink { } } +/** + * What `PictureReader` hands back: limited range BT.709, its values sRGB as drawn, the same as + * Chrome's encoder says of a recording of the whole editor. + */ +const PICTURE_COLOUR: VideoColorSpaceInit = { + primaries: 'bt709', + transfer: 'iec61966-2-1', + matrix: 'bt709', + fullRange: false, +}; + /** * The game view, drawn into a texture of the recording's size and read back from the GPU - * (`PictureReader`): the frames come out a tick or two after they are taken. + * (`PictureReader`) as I420: the frames come out a tick or two after they are taken. */ class ViewSource implements FrameSource { private reader: PictureReader | null = null; @@ -170,12 +181,15 @@ class ViewSource implements FrameSource { } collect(encode: (frame: VideoFrame) => void): void { + const layout = this.reader?.layout.layout; this.reader?.collect((bytes, ms) => encode( new VideoFrame(bytes, { - format: 'RGBX', + format: 'I420', codedWidth: this.size.w, codedHeight: this.size.h, + layout, + colorSpace: PICTURE_COLOUR, timestamp: Math.round(ms * 1000), duration: Math.round(1e6 / this.settings.fps), }), diff --git a/src/viewport/PictureReader.ts b/src/viewport/PictureReader.ts index c4426501..78118cf4 100644 --- a/src/viewport/PictureReader.ts +++ b/src/viewport/PictureReader.ts @@ -1,9 +1,70 @@ import { RenderTexture, type WebGLRenderer } from 'pixi.js'; +import { i420Layout } from '../core/recording/recording'; import { ui } from '../ui'; /** Frames on their way back from the GPU at once; while all are, a frame due is dropped. */ const MOST_IN_FLIGHT = 3; +/** A triangle over the whole target, from the vertex index alone. */ +const VERTEX = `#version 300 es +void main() { + vec2 corner = vec2(float((gl_VertexID << 1) & 2), float(gl_VertexID & 2)); + gl_Position = vec4(corner * 2.0 - 1.0, 0.0, 1.0); +}`; + +/** + * The picture as I420 in limited range BT.709, 4 bytes a texel in the layout of `i420Layout`. + * Texel row 0 is the top of the picture, as Pixi draws a render texture; a chroma sample is the + * mean of its 2 × 2 pixels. + */ +const FRAGMENT = `#version 300 es +precision highp float; +precision highp int; +uniform sampler2D picture; +uniform ivec2 size; +uniform int stride; +uniform int chromaRows; +out vec4 colour; + +const vec3 LUMA = vec3(0.2126, 0.7152, 0.0722); + +vec3 rgbAt(int x, int y) { + return texelFetch(picture, ivec2(min(x, size.x - 1), min(y, size.y - 1)), 0).rgb; +} + +float luma(vec3 c) { + return (16.0 + 219.0 * dot(c, LUMA)) / 255.0; +} + +float chroma(vec3 c, int plane) { + float y = dot(c, LUMA); + float d = plane == 0 ? (c.b - y) / 1.8556 : (c.r - y) / 1.5748; + return (128.0 + 224.0 * d) / 255.0; +} + +void main() { + ivec2 at = ivec2(gl_FragCoord.xy); + int x = at.x * 4; + if (at.y < size.y) { + colour = vec4(luma(rgbAt(x, at.y)), luma(rgbAt(x + 1, at.y)), luma(rgbAt(x + 2, at.y)), luma(rgbAt(x + 3, at.y))); + return; + } + int row = at.y - size.y; + int plane = row / chromaRows; + int byte = (row - plane * chromaRows) * stride + x; + int chromaStride = stride / 2; + int cy = byte / chromaStride; + int cx = byte - cy * chromaStride; + vec4 four; + for (int k = 0; k < 4; k++) { + int px = 2 * (cx + k); + int py = 2 * cy; + vec3 c = rgbAt(px, py) + rgbAt(px + 1, py) + rgbAt(px, py + 1) + rgbAt(px + 1, py + 1); + four[k] = chroma(c * 0.25, plane); + } + colour = four; +}`; + interface InFlight { buffer: WebGLBuffer; fence: WebGLSync; @@ -15,13 +76,26 @@ interface InFlight { * canvas shows what is drawn into it at once, so a frame drawn there flickered in its corner. The * view is drawn into `texture` instead and read back through pixel buffers, so the main thread does * not wait for the GPU: a frame comes out of `collect` a tick or two after it was drawn. + * + * What comes back is I420 in limited range BT.709, turned so on the GPU: fed RGB, Chrome's encoder + * made full-range video, which VLC 3 shows too dark whatever the file says. It is also less than + * half the bytes of RGBA to read back. */ export class PictureReader { readonly texture: RenderTexture; + readonly layout: ReturnType; private readonly gl: WebGL2RenderingContext; private readonly free: WebGLBuffer[] = []; private readonly inFlight: InFlight[] = []; private readonly bytes: Uint8Array; + private readonly program: WebGLProgram; + private readonly uniforms: Record< + 'picture' | 'size' | 'stride' | 'chromaRows', + WebGLUniformLocation | null + >; + private readonly vertices: WebGLVertexArrayObject; + private readonly planes: WebGLTexture; + private readonly framebuffer: WebGLFramebuffer; constructor( readonly w: number, @@ -32,14 +106,31 @@ export class PictureReader { if (!gl || typeof gl.fenceSync !== 'function') throw new Error('recording needs WebGL 2'); this.gl = gl; this.texture = RenderTexture.create({ width: w, height: h, resolution: 1, antialias: true }); + this.layout = i420Layout(w, h); + const { texels } = this.layout; + this.program = linkProgram(gl); + this.uniforms = { + picture: gl.getUniformLocation(this.program, 'picture'), + size: gl.getUniformLocation(this.program, 'size'), + stride: gl.getUniformLocation(this.program, 'stride'), + chromaRows: gl.getUniformLocation(this.program, 'chromaRows'), + }; + this.vertices = gl.createVertexArray()!; + this.planes = gl.createTexture()!; + gl.bindTexture(gl.TEXTURE_2D, this.planes); + gl.texStorage2D(gl.TEXTURE_2D, 1, gl.RGBA8, texels.w, texels.h); + this.framebuffer = gl.createFramebuffer()!; + gl.bindFramebuffer(gl.FRAMEBUFFER, this.framebuffer); + gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, this.planes, 0); for (let i = 0; i < MOST_IN_FLIGHT; i++) { const buffer = gl.createBuffer(); gl.bindBuffer(gl.PIXEL_PACK_BUFFER, buffer); - gl.bufferData(gl.PIXEL_PACK_BUFFER, w * h * 4, gl.STREAM_READ); + gl.bufferData(gl.PIXEL_PACK_BUFFER, this.layout.bytes, gl.STREAM_READ); this.free.push(buffer); } gl.bindBuffer(gl.PIXEL_PACK_BUFFER, null); - this.bytes = new Uint8Array(w * h * 4); + this.bytes = new Uint8Array(this.layout.bytes); + renderer.resetState(); } /** Whether a frame drawn now would have no buffer to go back in. */ @@ -60,20 +151,35 @@ export class PictureReader { const target = renderer.renderTarget.getRenderTarget(this.texture); // Resolves the multisampled picture into the texture; a second time is only a copy. renderer.renderTarget.adaptor.finishRenderPass(target); - const gpu = renderer.renderTarget.getGpuRenderTarget(target); - // Through the adaptor, so Pixi knows which framebuffer is bound. - renderer.renderTarget.adaptor.bindFramebuffer(gpu.resolveTargetFramebuffer); + const picture = renderer.texture.getGlSource(this.texture.source).texture; + const { texels, stride, chromaRows } = this.layout; + gl.bindFramebuffer(gl.FRAMEBUFFER, this.framebuffer); + gl.viewport(0, 0, texels.w, texels.h); + for (const flag of [gl.BLEND, gl.DEPTH_TEST, gl.STENCIL_TEST, gl.SCISSOR_TEST, gl.CULL_FACE]) + gl.disable(flag); + gl.colorMask(true, true, true, true); + gl.useProgram(this.program); + gl.bindVertexArray(this.vertices); + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, picture); + gl.uniform1i(this.uniforms.picture, 0); + gl.uniform2i(this.uniforms.size, this.w, this.h); + gl.uniform1i(this.uniforms.stride, stride); + gl.uniform1i(this.uniforms.chromaRows, chromaRows); + gl.drawArrays(gl.TRIANGLES, 0, 3); gl.bindBuffer(gl.PIXEL_PACK_BUFFER, buffer); - gl.readPixels(0, 0, this.w, this.h, gl.RGBA, gl.UNSIGNED_BYTE, 0); + gl.readPixels(0, 0, texels.w, texels.h, gl.RGBA, gl.UNSIGNED_BYTE, 0); gl.bindBuffer(gl.PIXEL_PACK_BUFFER, null); const fence = gl.fenceSync(gl.SYNC_GPU_COMMANDS_COMPLETE, 0)!; gl.flush(); this.inFlight.push({ buffer, fence, ms }); + // Pixi caches what it bound last; this tells it everything may have changed. + renderer.resetState(); } /** - * Hands over each frame the GPU has finished, oldest first, its rows top first as Pixi draws a - * texture. The bytes are reused for the next frame, so `take` copies what it keeps. + * Hands over each frame the GPU has finished, oldest first, as I420 in `layout`. The bytes are + * reused for the next frame, so `take` copies what it keeps. */ collect(take: (bytes: Uint8Array, ms: number) => void): void { const gl = this.gl; @@ -97,6 +203,33 @@ export class PictureReader { gl.deleteBuffer(buffer); } for (const buffer of this.free.splice(0)) gl.deleteBuffer(buffer); + gl.deleteFramebuffer(this.framebuffer); + gl.deleteTexture(this.planes); + gl.deleteVertexArray(this.vertices); + gl.deleteProgram(this.program); + (ui().app.renderer as WebGLRenderer).resetState(); this.texture.destroy(true); } } + +function linkProgram(gl: WebGL2RenderingContext): WebGLProgram { + const program = gl.createProgram(); + for (const [type, source] of [ + [gl.VERTEX_SHADER, VERTEX], + [gl.FRAGMENT_SHADER, FRAGMENT], + ] as const) { + const shader = gl.createShader(type)!; + gl.shaderSource(shader, source); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + throw new Error(`recording shader does not compile: ${gl.getShaderInfoLog(shader)}`); + } + gl.attachShader(program, shader); + gl.deleteShader(shader); + } + gl.linkProgram(program); + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + throw new Error(`recording shader does not link: ${gl.getProgramInfoLog(program)}`); + } + return program; +} diff --git a/src/viewport/RecordingTake.ts b/src/viewport/RecordingTake.ts index 7aa53fab..25269f74 100644 --- a/src/viewport/RecordingTake.ts +++ b/src/viewport/RecordingTake.ts @@ -1,6 +1,13 @@ import { VIDEO_WAIT_MS } from '../core/model/video'; import { preSkipOf } from '../core/music/ogg'; -import { AVC_CODEC, colourOf, MatroskaWriter, VP9_CODEC } from '../core/recording/matroska'; +import { avcConfigWithColour } from '../core/recording/avcColour'; +import { + AVC_CODEC, + colourOf, + type MatroskaColour, + MatroskaWriter, + VP9_CODEC, +} from '../core/recording/matroska'; import { avcCodec, FrameGate, @@ -391,14 +398,16 @@ export class Take { return; } const frame = 1000 / this.fps; + const avc = this.codec.startsWith('avc1'); + const colour = colourOf(this.colour); const { head, rest } = writer.finish({ video: { - codec: this.codec.startsWith('avc1') ? AVC_CODEC : VP9_CODEC, + codec: avc ? AVC_CODEC : VP9_CODEC, width: this.size.w, height: this.size.h, - private: this.videoHead, + private: avc ? this.avcHead(colour) : this.videoHead, fps: this.fps, - colour: colourOf(this.colour), + colour, }, audio: this.audio ? { @@ -426,6 +435,20 @@ export class Take { }); } + /** + * The encoder's `avcC` with the colour in its SPS too: Chrome leaves it out of the stream for some + * inputs, and a player that reads only the stream shows a full-range picture too dark. An SPS + * this cannot read keeps the file as the encoder made it. + */ + private avcHead(colour: MatroskaColour): Uint8Array | null { + if (!this.videoHead) return null; + try { + return avcConfigWithColour(this.videoHead, colour); + } catch { + return this.videoHead; + } + } + /** Takes the frames still on their way to the encoder, a moment at most. */ private async lastFrames(): Promise { const until = performance.now() + LAST_FRAMES_MS; diff --git a/tests/core/avc-colour.test.ts b/tests/core/avc-colour.test.ts new file mode 100644 index 00000000..43c729a3 --- /dev/null +++ b/tests/core/avc-colour.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from 'vitest'; +import { + avcConfigSps, + avcConfigWithColour, + spsColour, + spsWithColour, +} from '../../src/core/recording/avcColour'; +import type { MatroskaColour } from '../../src/core/recording/matroska'; + +const hex = (text: string) => Uint8Array.from(text.match(/../g)!.map((pair) => parseInt(pair, 16))); +const toHex = (bytes: Uint8Array) => [...bytes].map((value) => value.toString(16).padStart(2, '0')).join(''); + +/** Chrome 136 on Windows, a game view recording (RGBX frames): a VUI, but no `video_signal_type`. */ +const GAME_VIEW = hex( + '0164002affe100196764002aac2b403c0113f2e022000007d00003a9801e38554001000468ee3cb0fdf8f800', +); +/** The same file after the rewrite, as ffmpeg's `trace_headers` read it: full range, BT.709, sRGB. */ +const GAME_VIEW_FIXED = hex( + '0164002affe1001c6764002aac2b403c0113f2e02dc043405000003e80001d4c00f1c2aa01000468ee3cb0fdf8f800', +); +/** Chrome 136 on Windows, a recording of the editor (canvas frames): the colour already in the SPS. */ +const EDITOR = hex('01640c33ffe1001367640c33ac18d0050016b4d40434041e1108d401000468ce3c80fdf8f800'); +/** A baseline SPS without a VUI: 320×240, written by hand. */ +const BASELINE = hex('6742c01eda0507e4'); + +const FULL_SRGB: MatroskaColour = { matrix: 1, transfer: 13, primaries: 1, fullRange: true }; +const TV_SRGB: MatroskaColour = { matrix: 1, transfer: 13, primaries: 1, fullRange: false }; + +describe('avcConfigWithColour', () => { + it('writes the colour into an SPS that lacks it, as ffmpeg reads it back', () => { + expect(spsColour(avcConfigSps(GAME_VIEW)[0]!)).toBeNull(); + const fixed = avcConfigWithColour(GAME_VIEW, FULL_SRGB); + expect(toHex(fixed)).toBe(toHex(GAME_VIEW_FIXED)); + expect(spsColour(avcConfigSps(fixed)[0]!)).toEqual(FULL_SRGB); + }); + + it('keeps everything around the SPS byte for byte', () => { + const fixed = avcConfigWithColour(GAME_VIEW, FULL_SRGB); + expect(toHex(fixed.subarray(0, 6))).toBe(toHex(GAME_VIEW.subarray(0, 6))); + const pps = (config: Uint8Array) => config.subarray(6 + 2 + avcConfigSps(config)[0]!.length); + expect(toHex(pps(fixed))).toBe(toHex(pps(GAME_VIEW))); + }); + + it('changes nothing where the SPS already says the same colour', () => { + expect(spsColour(avcConfigSps(EDITOR)[0]!)).toEqual(TV_SRGB); + expect(toHex(avcConfigWithColour(EDITOR, TV_SRGB))).toBe(toHex(EDITOR)); + }); + + it('replaces a colour the SPS says with the one asked for', () => { + const changed = avcConfigWithColour(EDITOR, FULL_SRGB); + expect(spsColour(avcConfigSps(changed)[0]!)).toEqual(FULL_SRGB); + expect(toHex(avcConfigWithColour(changed, TV_SRGB))).toBe(toHex(EDITOR)); + }); + + it('is the same once written twice', () => { + const once = avcConfigWithColour(GAME_VIEW, FULL_SRGB); + expect(toHex(avcConfigWithColour(once, FULL_SRGB))).toBe(toHex(once)); + }); + + it('refuses what is no avcC', () => { + expect(() => avcConfigWithColour(hex('00000001'), FULL_SRGB)).toThrow(); + }); +}); + +describe('spsWithColour', () => { + it('gives an SPS without a VUI one that says the colour', () => { + expect(spsColour(BASELINE)).toBeNull(); + const fixed = spsWithColour(BASELINE, FULL_SRGB); + // As ffmpeg's `trace_headers` reads it: 320×240 still, then the VUI with the colour alone. + expect(toHex(fixed)).toBe('6742c01eda0507e9b808680810'); + expect(spsColour(fixed)).toEqual(FULL_SRGB); + }); + + it('escapes what would read as a start code, and reads it back', () => { + // Zeros in all three numbers put 00 00 00 into this SPS, which has to go out as 00 00 03 00. + const zero: MatroskaColour = { matrix: 0, transfer: 0, primaries: 0, fullRange: false }; + const escaped = avcConfigWithColour(EDITOR, zero); + expect(toHex(avcConfigSps(escaped)[0]!)).toBe('67640c33ac18d0050016b4d4000003001e1108d4'); + expect(spsColour(avcConfigSps(escaped)[0]!)).toEqual(zero); + expect(toHex(avcConfigWithColour(escaped, TV_SRGB))).toBe(toHex(EDITOR)); + }); + + it('refuses a NAL unit that is no SPS', () => { + expect(() => spsWithColour(hex('68ee3cb0'), FULL_SRGB)).toThrow(); + }); +}); diff --git a/tests/core/recording.test.ts b/tests/core/recording.test.ts index eb6e4b84..3b90b705 100644 --- a/tests/core/recording.test.ts +++ b/tests/core/recording.test.ts @@ -16,6 +16,7 @@ import { elapsedText, fitPicture, FrameGate, + i420Layout, isKeyFrame, pictureTransform, RecordingClock, @@ -77,6 +78,30 @@ describe('recording settings (#405)', () => { expect(recordingSize('1440p', { w: 2100, h: 900 }, { w: 2100, h: 900 })).toEqual({ w: 3360, h: 1440 }); }); + it('lays the I420 planes of a frame read back from the GPU out on whole texels', () => { + // 1920 is a multiple of 8, 1080 / 2 rows of chroma fill 270 texel rows a plane. + expect(i420Layout(1920, 1080)).toEqual({ + stride: 1920, + chromaRows: 270, + texels: { w: 480, h: 1620 }, + bytes: 1920 * 1620, + layout: [ + { offset: 0, stride: 1920 }, + { offset: 1920 * 1080, stride: 960 }, + { offset: 1920 * 1350, stride: 960 }, + ], + }); + // 1646 is not: a luma row takes 1648 bytes. 722 / 2 = 361 chroma rows, odd, so the last texel + // row of each plane is half used. + const odd = i420Layout(1646, 722); + expect(odd.stride).toBe(1648); + expect(odd.chromaRows).toBe(181); + expect(odd.texels).toEqual({ w: 412, h: 722 + 362 }); + expect(odd.layout[2]!.offset - odd.layout[1]!.offset).toBe(1648 * 181); + expect(odd.layout[1]!.stride * 361).toBeLessThanOrEqual(1648 * 181); + expect(odd.bytes).toBe(odd.layout[2]!.offset + 1648 * 181); + }); + it('picks the lowest H.264 level that takes the frames', () => { expect(avcCodec(1280, 720, 30)).toBe('avc1.64001f'); expect(avcCodec(1920, 1080, 30)).toBe('avc1.640028'); diff --git a/tools/e2e/67-recording.mjs b/tools/e2e/67-recording.mjs index 69e1b565..a9bc449f 100644 --- a/tools/e2e/67-recording.mjs +++ b/tools/e2e/67-recording.mjs @@ -1,3 +1,7 @@ +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { MAKE_OBJECT, pickAssetType } from './lib/scaffold.mjs'; /** @@ -196,6 +200,58 @@ export async function recording(t) { colors, ); + // 67d-ii. The original is limited range and says so in the H.264 stream too, not only in the + // Matroska head: VLC 3 showed a full-range recording too dark, even with the range in the + // stream. ffprobe reads the file, then the stream taken out of it as raw H.264. + const original = named + ? await fetch(`${t.baseUrl}data/videos/${named.file.replace(/\.mp4$/, '.source.mkv')}`) + : null; + const streamColour = { file: null, stream: null }; + if (original?.ok) { + const dir = mkdtempSync(join(tmpdir(), 'rc67-')); + const mkv = join(dir, 'take.mkv'); + const raw = join(dir, 'take.h264'); + writeFileSync(mkv, new Uint8Array(await original.arrayBuffer())); + execFileSync('ffmpeg', [ + '-v', + 'error', + '-y', + '-i', + mkv, + '-map', + '0:v', + '-c', + 'copy', + '-bsf:v', + 'h264_mp4toannexb', + '-f', + 'h264', + raw, + ]); + const probe = (path) => + execFileSync('ffprobe', [ + '-v', + 'error', + '-select_streams', + 'v:0', + '-of', + 'json', + '-show_entries', + 'stream=color_range,color_space,color_transfer,color_primaries', + path, + ]).toString(); + streamColour.file = JSON.parse(probe(mkv)).streams[0]; + streamColour.stream = JSON.parse(probe(raw)).streams[0]; + rmSync(dir, { recursive: true, force: true }); + } + check( + 'the recording is limited range and says its colour in the H.264 stream as in its Matroska head', + streamColour.file !== null && + streamColour.file.color_range === 'tv' && + JSON.stringify(streamColour.stream) === JSON.stringify(streamColour.file), + JSON.stringify({ status: original?.status ?? null, ...streamColour }), + ); + // 67e. The editor records itself from the main menu: the whole canvas at 60 fps, the mouse pointer // drawn in where the system draws it over the canvas, and the red dot beside the menu on // screen only. The frame is read where the pointer rested on the empty middle of the top bar.