From d48f174caf6ce1593593aa5fcc73138f4d644bdc Mon Sep 17 00:00:00 2001 From: MelekTaus Date: Sun, 11 Oct 2026 12:12:39 +0200 Subject: [PATCH] Play videos on sprites, converted on the server with clips (#402) A video becomes an asset like a sound: uploaded once, converted to one format by a worker of its own, cut into clips in its tab, and worn by a sprite through its Texture row. Only Chrome is a target, without alpha. - Server: POST ?r=videos queues the upload under the hash of its bytes, so the MP4's name is known at once. The worker (VideoWorker over VideoFormat) writes H.264/AAC, yuv420p, constant frame rate, a keyframe every 15 frames, at most 1920x1080, with a poster and the details beside it. A failed job says why and is tried again on the next upload. - An upload may be up to 2 GB. It is streamed from the request to disk and hashed there, so php-fpm holds none of it in memory. The other routes keep their 256 MB. - The worker runs in the container game-maker-video on stein2, a compose project of its own (tools/video/), and in CI beside the PHP server. The CI image has ffmpeg for it. - Model: asset type video with size, length, frame rate and sound, and clips like a sound's. The clip helpers and commands serve both. - A sprite's Texture row offers videos and clips after the pictures. resolvePicture names a playing video by a key, and the texture library hands it to a VideoLibrary with one player per key. - The editor shows a placed video as a quiet preview. Play starts it over at its clip's start and lets it be heard through the master bus, a pause holds it, Esc makes it quiet again. - Video tab: the picture, a ruler, the clip strip and a track on the Sound tab's time axis. A drag spans a clip, a handle moves an edge, Play plays the picked clip. - The picker shows a video on its way with its progress and adopts an upload from it once it is ready. Dropping a video file does the same. Decided on my own: - Two sprites wearing the same clip share one player and show the same picture. Control per instance belongs to the follow-up with nodes. - A video plays in real time, the game's time scale does not slow it. - The container may use 3 of the 4 cores of stein2. Findings: - The first real upload (279 MB) got 413: the API's own ceiling of 256 MB for every request, not nginx or the Brandmauer. - Chrome has no picture of a video that never played, and a Pixi VideoSource drawn before the first frame stays empty for good. The texture is made in the first frame callback (openVideo). - A player nobody asks for in 2 s is let go. The inspector's PREVIEW asked only on an edit, so with the sprite not on screen its texture was destroyed under it and the batcher threw on every frame. The preview now redraws every frame while it shows a video, like a Time shader. A let-go texture keeps an empty source instead of none. - VideoSource.load() reloads the element and cuts off an earlier play(). Playing starts after loadeddata. Tests: - PHP videos.test.php: 68 checks, with the real ffmpeg - 300 MB upload through Brandmauer and nginx: 201 in 21 s, php-fpm at 13.7 MB RSS at most - Unit videos.test.ts: 20 tests - e2e 66 (stein2, atx was off): ALL PASSED, 18 checks; e2e 04 with the new asset count green - Gate (atx): 294 files, 4520 tests. serialize.test.ts used video as its example of an unknown type and now uses a made-up one. - Counter-checks: a keyframe gap of 30 fails the PHP check, a key that keeps the video's end fails 2 unit tests, a clip without its end fails "plays only its stretch" in e2e 66 Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/ci.yml | 1 + .gitignore | 1 + README.md | 2 +- api/index.php | 65 ++- api/lib/VideoFormat.php | 216 +++++++++ api/lib/VideoQueue.php | 173 +++++++ api/lib/VideoStore.php | 234 ++++++++++ api/lib/VideoWorker.php | 150 ++++++ src/core/commands/assetCommands.ts | 23 + src/core/commands/clipCommands.ts | 51 ++- src/core/model/Asset.ts | 28 +- src/core/model/AssetTypes.ts | 7 +- src/core/model/sound.ts | 34 +- src/core/model/texture.ts | 22 +- src/core/model/video.ts | 110 +++++ src/core/properties/PropertySchema.ts | 9 +- src/core/properties/textureRef.ts | 36 +- src/core/properties/videos.ts | 36 ++ src/core/store/HttpFileStore.ts | 5 +- src/core/store/VideoStore.ts | 28 ++ src/core/store/serialize.ts | 32 +- src/editor/App.ts | 2 + src/editor/Editor.ts | 15 +- src/editor/PlaySession.ts | 4 + src/editor/actions.ts | 7 +- src/editor/assetImport.ts | 2 +- src/editor/assetTabs.ts | 1 + src/editor/debug/videoHooks.ts | 40 ++ src/editor/debugHook.ts | 2 + src/editor/fileImport.ts | 21 +- src/editor/inspector/SpritePreview.ts | 5 +- src/editor/inspector/clipSection.ts | 48 +- src/editor/inspector/fieldFactory.ts | 13 +- src/editor/panels/AssetPanel.ts | 21 +- src/editor/panels/assetGroups.ts | 1 + src/editor/panels/assetIcons.ts | 2 + src/editor/panels/icons/video.svg | 6 + src/editor/shader/ShaderEditor.ts | 1 + src/editor/shader/ShaderPreview.ts | 7 + src/editor/state.ts | 20 +- src/editor/video/VideoCanvas.ts | 633 ++++++++++++++++++++++++++ src/editor/video/VideoEditor.ts | 84 ++++ src/editor/video/VideoPicker.ts | 185 ++++++++ src/editor/video/videoInfo.ts | 36 ++ src/editor/videoImport.ts | 77 ++++ src/player/Player.ts | 3 + src/viewport/SoundLibrary.ts | 14 + src/viewport/TextureLibrary.ts | 9 +- src/viewport/VideoLibrary.ts | 259 +++++++++++ src/viewport/WorldViewport.ts | 5 +- tests/core/serialize.test.ts | 4 +- tests/core/videos.test.ts | 237 ++++++++++ tests/php/videos.test.php | 219 +++++++++ tools/ci/Dockerfile | 3 + tools/ci/serve.sh | 3 + tools/e2e/04-assets.mjs | 2 +- tools/e2e/66-video.mjs | 233 ++++++++++ tools/e2e/fixtures/colors.webm | Bin 0 -> 10279 bytes tools/e2e/index.mjs | 2 + tools/lib/projects.mjs | 33 ++ tools/video-worker.php | 48 ++ tools/video/Dockerfile | 12 + tools/video/compose.yml | 21 + 63 files changed, 3500 insertions(+), 103 deletions(-) create mode 100644 api/lib/VideoFormat.php create mode 100644 api/lib/VideoQueue.php create mode 100644 api/lib/VideoStore.php create mode 100644 api/lib/VideoWorker.php create mode 100644 src/core/model/video.ts create mode 100644 src/core/properties/videos.ts create mode 100644 src/core/store/VideoStore.ts create mode 100644 src/editor/debug/videoHooks.ts create mode 100644 src/editor/panels/icons/video.svg create mode 100644 src/editor/video/VideoCanvas.ts create mode 100644 src/editor/video/VideoEditor.ts create mode 100644 src/editor/video/VideoPicker.ts create mode 100644 src/editor/video/videoInfo.ts create mode 100644 src/editor/videoImport.ts create mode 100644 src/viewport/VideoLibrary.ts create mode 100644 tests/core/videos.test.ts create mode 100644 tests/php/videos.test.php create mode 100644 tools/e2e/66-video.mjs create mode 100644 tools/e2e/fixtures/colors.webm create mode 100644 tools/video-worker.php create mode 100644 tools/video/Dockerfile create mode 100644 tools/video/compose.yml diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 46b889d5..e9ba9634 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -43,6 +43,7 @@ jobs: php tests/php/snapshots.test.php php tests/php/releases.test.php php tests/php/files.test.php + php tests/php/videos.test.php suite: runs-on: game-maker-wsl diff --git a/.gitignore b/.gitignore index 0f971aa8..b3b65338 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ /data/textures/ /data/fonts/ /data/sounds/ +/data/videos/ /data/pixels/ /data/plugins/ /dist/ diff --git a/README.md b/README.md index 7d64e470..736484ff 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh The browser suite and the three example games need a machine with a GPU: ```bash -node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 1976 checks (GM_STEPS=20 for one step) +node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 1994 checks (GM_STEPS=20 for one step) # every tool signs in first – see the Accounts page node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs diff --git a/api/index.php b/api/index.php index d211942f..bb102a77 100644 --- a/api/index.php +++ b/api/index.php @@ -63,6 +63,13 @@ declare(strict_types=1); * POST ?r=sounds&name=x.wav raw sound bytes → {file,name,bytes} * DELETE ?r=sounds/{file} → 204 | 403 | 404 | 409 * + * GET ?r=videos → [{file,name,status,…,owner,deletable}] + * POST ?r=videos&name=x.mov raw video bytes → {file,name,status,…} + * DELETE ?r=videos/{file} → 204 | 403 | 404 | 409 + * (an upload is converted by the worker in `game-maker-video` (#402): + * `status` is queued, converting with `progress`, failed with `error`, + * or ready with width, height, duration, fps, audio and `poster`) + * * Uploads land in the uploader's own folder, and `file` carries it: `{folder}/{sha256}.{ext}` * (#344). A `file` without a folder belongs to nobody. A list shows the files of the one asking and * nobody's, an administrator's every file; `owner` is the account a file belongs to (null for @@ -78,8 +85,8 @@ declare(strict_types=1); * empty when none is known. It is a caption, never a path: the stored file is always `{hash}.{ext}` * in a folder. * - * Stored images, fonts, sounds and cels are served straight from `data/textures/`, `data/fonts/`, - * `data/sounds/` and `data/pixels/` as static files; the API only writes them. + * Stored images, fonts, sounds, videos and cels are served straight from `data/textures/`, `data/fonts/`, + * `data/sounds/`, `data/videos/` and `data/pixels/` as static files; the API only writes them. */ require __DIR__ . '/lib/AccountRules.php'; @@ -99,8 +106,13 @@ require __DIR__ . '/lib/SoundStore.php'; require __DIR__ . '/lib/Store.php'; require __DIR__ . '/lib/TextureStore.php'; require __DIR__ . '/lib/UserStore.php'; +require __DIR__ . '/lib/VideoQueue.php'; +require __DIR__ . '/lib/VideoStore.php'; -/** Has to clear the largest single upload below, which is a sound; the web server has its own say. */ +/** + * Has to clear the largest upload held in memory, which is a sound; a video is streamed to disk and + * has its own ceiling. The web server and the proxy in front of it have their own say. + */ const MAX_BODY_BYTES = 256 * 1024 * 1024; /** Upload ceiling for a single image. The editor downscales larger pictures before sending. */ @@ -112,6 +124,12 @@ const MAX_FONT_BYTES = 4 * 1024 * 1024; /** The same for a sound, far higher: a recorded take is plain WAV, about 46 minutes in mono. */ const MAX_SOUND_BYTES = 256 * 1024 * 1024; +/** + * A video goes up as it was recorded and is made smaller by the worker, not on the way. It is + * streamed to disk, so memory is no limit; nginx and the Brandmauer let 2 GB through (#402). + */ +const MAX_VIDEO_BYTES = 2048 * 1024 * 1024; + set_error_handler(static function (int $severity, string $message, string $file, int $line): bool { // PHP calls the handler for a call silenced with `@` as well, only with less reported: that // warning was expected and is answered where it happens (an unreadable image is a 422). @@ -514,12 +532,21 @@ function routeFiles( Response::json($store->list($users->folderOf($user), $isAdmin, $users->folderOwners())); } if ($method === 'POST') { - $bytes = Response::rawBody(); - if (strlen($bytes) > $maxBytes) { - Response::error("{$noun} must be at most " . ($maxBytes / 1024 / 1024) . ' MB', 413); - } + $tooLarge = "{$noun} must be at most " . ($maxBytes / 1024 / 1024) . ' MB'; + $name = (string) ($_GET['name'] ?? ''); try { - Response::json($store->store($bytes, (string) ($_GET['name'] ?? ''), $users->folderOf($user, true)), 201); + // A video is too big for memory and goes from the request straight to disk. + if ($store instanceof VideoStore) { + $in = fopen('php://input', 'rb'); + Response::json($store->storeStream($in, $maxBytes, $name, $users->folderOf($user, true)), 201); + } + $bytes = Response::rawBody(); + if (strlen($bytes) > $maxBytes) { + Response::error($tooLarge, 413); + } + Response::json($store->store($bytes, $name, $users->folderOf($user, true)), 201); + } catch (VideoTooLargeException) { + Response::error($tooLarge, 413); } catch (RuntimeException $e) { if (!$e instanceof $invalid) { throw $e; @@ -600,19 +627,20 @@ function routePlugins(array $parts, string $method): never } try { - if ((int) ($_SERVER['CONTENT_LENGTH'] ?? 0) > MAX_BODY_BYTES) { - Response::error('Request body too large', 413); - } - $method = $_SERVER['REQUEST_METHOD'] ?? 'GET'; $route = trim((string) ($_GET['r'] ?? ''), '/'); $parts = $route === '' ? [] : explode('/', $route); + $maxBody = ($parts[0] ?? '') === 'videos' ? MAX_VIDEO_BYTES : MAX_BODY_BYTES; + if ((int) ($_SERVER['CONTENT_LENGTH'] ?? 0) > $maxBody) { + Response::error('Request body too large', 413); + } + // A project reaches deepest, `projects/{id}/snapshots/{snapshotId}/restore`; a stored file is // `textures/{folder}/{file}`; everything else stops at two parts. $depth = match ($parts[0] ?? '') { 'projects' => 5, - 'textures', 'fonts', 'sounds' => 3, + 'textures', 'fonts', 'sounds', 'videos' => 3, default => 2, }; if (count($parts) < 1 || count($parts) > $depth) { @@ -671,6 +699,17 @@ try { 'Sound', InvalidSoundException::class, ), + 'videos' => routeFiles( + $parts, + $method, + $user, + $isAdmin, + $users, + new VideoStore("{$data}/videos"), + MAX_VIDEO_BYTES, + 'Video', + InvalidVideoException::class, + ), 'pixels' => routePixels($parts, $method), 'plugins' => routePlugins($parts, $method), default => Response::error('Not found', 404), diff --git a/api/lib/VideoFormat.php b/api/lib/VideoFormat.php new file mode 100644 index 00000000..c1e57883 --- /dev/null +++ b/api/lib/VideoFormat.php @@ -0,0 +1,216 @@ + $probe ffprobe's `-show_streams -show_format` as JSON + * @return array{width:int,height:int,fps:string,audio:bool,duration:float} + * @throws InvalidArgumentException when there is no picture in it + */ + public static function plan(array $probe): array + { + $streams = is_array($probe['streams'] ?? null) ? $probe['streams'] : []; + $video = null; + $audio = false; + foreach ($streams as $stream) { + $type = $stream['codec_type'] ?? ''; + // A cover picture in a music file is a video stream as well, one that is a single frame. + $still = (int) (($stream['disposition'] ?? [])['attached_pic'] ?? 0) === 1; + if ($type === 'video' && $video === null && !$still) { + $video = $stream; + } + $audio = $audio || $type === 'audio'; + } + if ($video === null || (int) ($video['width'] ?? 0) <= 0 || (int) ($video['height'] ?? 0) <= 0) { + throw new InvalidArgumentException('There is no picture in this file'); + } + + [$width, $height] = [(int) $video['width'], (int) $video['height']]; + if (abs(self::rotation($video)) % 180 === 90) { + [$width, $height] = [$height, $width]; + } + [$width, $height] = self::fit($width, $height); + + return [ + 'width' => $width, + 'height' => $height, + 'fps' => self::fps($video), + 'audio' => $audio, + 'duration' => self::duration($probe, $video), + ]; + } + + /** + * The size inside the box, keeping the shape, both sides even (H.264 in `yuv420p` needs that). + * + * @return array{0:int,1:int} + */ + public static function fit(int $width, int $height): array + { + [$boxW, $boxH] = $width >= $height + ? [self::MAX_LONG_SIDE, self::MAX_SHORT_SIDE] + : [self::MAX_SHORT_SIDE, self::MAX_LONG_SIDE]; + $scale = min(1.0, $boxW / $width, $boxH / $height); + $even = static fn(float $side): int => max(2, 2 * (int) floor(round($side, 3) / 2)); + return [$even($width * $scale), $even($height * $scale)]; + } + + /** + * The frame rate the file is written at: its own average, unless that is above 60 or unknown. + * Kept as ffmpeg's fraction, so 29.97 stays 30000/1001 and frames do not drift against the time. + * + * @param array $video + */ + public static function fps(array $video): string + { + foreach (['avg_frame_rate', 'r_frame_rate'] as $key) { + $rate = self::rate((string) ($video[$key] ?? '')); + if ($rate !== null) { + return $rate[1] > self::MAX_FPS ? (string) self::MAX_FPS : $rate[0]; + } + } + return self::FALLBACK_FPS; + } + + /** A fraction such as `30000/1001` as a number of frames per second. */ + public static function fpsValue(string $fps): float + { + $rate = self::rate($fps); + return $rate === null ? 0.0 : $rate[1]; + } + + /** + * @param array{width:int,height:int,fps:string,audio:bool} $plan + * @return list + */ + public static function convertArgs(string $source, string $target, array $plan): array + { + $args = [ + 'ffmpeg', '-nostdin', '-hide_banner', '-loglevel', 'error', '-y', + '-i', $source, + '-map', '0:v:0', + ]; + if ($plan['audio']) { + $args = [...$args, '-map', '0:a:0']; + } + $args = [ + ...$args, + '-vf', "scale={$plan['width']}:{$plan['height']}:flags=lanczos,setsar=1", + '-r', $plan['fps'], '-fps_mode', 'cfr', + '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '21', + '-pix_fmt', 'yuv420p', + '-g', (string) self::KEYFRAME_EVERY, '-keyint_min', (string) self::KEYFRAME_EVERY, '-sc_threshold', '0', + ]; + $args = $plan['audio'] + ? [...$args, '-c:a', 'aac', '-b:a', '160k', '-ac', '2', '-ar', '48000'] + : [...$args, '-an']; + return [...$args, '-sn', '-dn', '-map_metadata', '-1', '-movflags', '+faststart', '-progress', 'pipe:1', '-nostats', '-f', 'mp4', $target]; + } + + /** + * A still from a little way in (not the very first frame, which is often black), as wide as the + * picker draws it. + * + * @return list + */ + public static function posterArgs(string $video, string $target, float $duration): array + { + $at = sprintf('%.3f', max(0.0, min(1.0, $duration / 2))); + return [ + 'ffmpeg', '-nostdin', '-hide_banner', '-loglevel', 'error', '-y', + '-ss', $at, '-i', $video, '-frames:v', '1', + '-vf', 'scale=320:-2', '-q:v', '4', '-f', 'image2', $target, + ]; + } + + /** @return list */ + public static function probeArgs(string $path): array + { + return ['ffprobe', '-v', 'error', '-print_format', 'json', '-show_streams', '-show_format', $path]; + } + + /** + * What the editor is told about a converted file, read back from the file itself rather than + * the plan: a crop or a rounding inside ffmpeg is what the editor has to draw. + * + * @param array $probe + * @return array{width:int,height:int,duration:float,fps:float,audio:bool} + */ + public static function details(array $probe): array + { + $plan = self::plan($probe); + return [ + 'width' => $plan['width'], + 'height' => $plan['height'], + 'duration' => round($plan['duration'], 3), + 'fps' => round(self::fpsValue($plan['fps']), 3), + 'audio' => $plan['audio'], + ]; + } + + /** The share of the work done, from one `out_time_us=` line of ffmpeg's progress. */ + public static function progress(string $line, float $duration): ?float + { + if ($duration <= 0 || !preg_match('/^out_time_us=(\d+)$/', trim($line), $m)) { + return null; + } + return max(0.0, min(1.0, ((int) $m[1]) / 1e6 / $duration)); + } + + /** @param array $video */ + private static function rotation(array $video): int + { + foreach (is_array($video['side_data_list'] ?? null) ? $video['side_data_list'] : [] as $side) { + if (isset($side['rotation']) && is_numeric($side['rotation'])) { + return (int) $side['rotation']; + } + } + $tag = ($video['tags'] ?? [])['rotate'] ?? null; + return is_numeric($tag) ? (int) $tag : 0; + } + + /** @param array $probe @param array $video */ + private static function duration(array $probe, array $video): float + { + foreach ([$video['duration'] ?? null, ($probe['format'] ?? [])['duration'] ?? null] as $value) { + if (is_numeric($value) && (float) $value > 0) { + return (float) $value; + } + } + return 0.0; + } + + /** @return array{0:string,1:float}|null the fraction as written and its value, null for none */ + private static function rate(string $text): ?array + { + if (!preg_match('#^(\d+)(?:/(\d+))?$#', $text, $m)) { + return null; + } + $num = (int) $m[1]; + $den = isset($m[2]) ? (int) $m[2] : 1; + if ($num <= 0 || $den <= 0) { + return null; + } + return [$den === 1 ? (string) $num : "{$num}/{$den}", $num / $den]; + } +} diff --git a/api/lib/VideoQueue.php b/api/lib/VideoQueue.php new file mode 100644 index 00000000..4ba76658 --- /dev/null +++ b/api/lib/VideoQueue.php @@ -0,0 +1,173 @@ +dir) && !mkdir($this->dir, 0775, true) && !is_dir($this->dir)) { + throw new RuntimeException("Cannot create directory {$this->dir}"); + } + } + + /** + * Copies an upload into a temp file beside the jobs, never more than `$maxBytes` of it: a video + * is too big to be held in memory. The caller hands the file to add() or deletes it. + * + * @param resource $in + * @throws VideoTooLargeException when the stream has more than `$maxBytes` + */ + public function receive($in, int $maxBytes): string + { + $tmp = "{$this->dir}/upload-" . bin2hex(random_bytes(8)) . '.tmp'; + $out = fopen($tmp, 'xb'); + if ($out === false) { + throw new RuntimeException('Cannot write uploaded video'); + } + $copied = stream_copy_to_stream($in, $out, $maxBytes + 1); + fclose($out); + if ($copied === false || $copied > $maxBytes) { + unlink($tmp); + throw $copied === false ? new RuntimeException('Cannot write uploaded video') : new VideoTooLargeException(); + } + return $tmp; + } + + /** Queues the upload received into `$upload`, which is moved and gone afterwards. */ + public function add(string $folder, string $hash, string $upload, string $name): void + { + if (!chmod($upload, AtomicFile::HIDDEN) || !rename($upload, $this->source($folder, $hash))) { + throw new RuntimeException('Cannot write uploaded video'); + } + $this->write($folder, $hash, [ + 'folder' => $folder, + 'hash' => $hash, + 'name' => $name, + 'state' => self::QUEUED, + 'progress' => 0, + 'error' => '', + 'createdAt' => gmdate('c'), + ]); + } + + /** @return array|null */ + public function job(string $folder, string $hash): ?array + { + $path = $this->jobPath($folder, $hash); + if (!is_file($path)) { + return null; + } + $raw = @file_get_contents($path); + $job = $raw === false ? null : json_decode($raw, true); + return is_array($job) ? $job : null; + } + + /** @return list> every job, oldest first */ + public function jobs(): array + { + $jobs = []; + foreach (glob($this->dir . '/*.json') ?: [] as $path) { + if (!preg_match('/^(' . UserStore::FOLDER_PATTERN . ')-([a-f0-9]{64})\.json$/', basename($path), $m)) { + continue; + } + $job = $this->job($m[1], $m[2]); + if ($job !== null) { + $jobs[] = $job; + } + } + usort($jobs, static fn(array $a, array $b): int => strcmp((string) $a['createdAt'], (string) $b['createdAt'])); + return $jobs; + } + + /** The oldest job nobody has started on, or null when there is none. */ + public function next(): ?array + { + foreach ($this->jobs() as $job) { + if ($job['state'] === self::QUEUED && is_file($this->source($job['folder'], $job['hash']))) { + return $job; + } + } + return null; + } + + /** + * Changes a job, unless it was deleted in the meantime: then false, and the one converting it + * knows to stop. + * + * @param array $changes + */ + public function update(string $folder, string $hash, array $changes): bool + { + $job = $this->job($folder, $hash); + if ($job === null) { + return false; + } + $this->write($folder, $hash, $changes + $job); + return true; + } + + /** A job that was being converted when the worker stopped is started again. */ + public function requeueInterrupted(): int + { + $count = 0; + foreach ($this->jobs() as $job) { + if ($job['state'] === self::CONVERTING) { + $this->update($job['folder'], $job['hash'], ['state' => self::QUEUED, 'progress' => 0]); + $count++; + } + } + return $count; + } + + public function remove(string $folder, string $hash): bool + { + $removed = false; + foreach ([$this->source($folder, $hash), $this->jobPath($folder, $hash)] as $path) { + if (is_file($path)) { + $removed = unlink($path) || $removed; + } + } + return $removed; + } + + /** Drops the upload once it is converted or cannot be; the job itself says what happened. */ + public function dropSource(string $folder, string $hash): void + { + $path = $this->source($folder, $hash); + if (is_file($path)) { + unlink($path); + } + } + + public function source(string $folder, string $hash): string + { + return "{$this->dir}/{$folder}-{$hash}.src"; + } + + private function jobPath(string $folder, string $hash): string + { + return "{$this->dir}/{$folder}-{$hash}.json"; + } + + /** @param array $job */ + private function write(string $folder, string $hash, array $job): void + { + $json = json_encode($job, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); + AtomicFile::write($this->jobPath($folder, $hash), (string) $json, AtomicFile::HIDDEN, 'video job'); + } +} diff --git a/api/lib/VideoStore.php b/api/lib/VideoStore.php new file mode 100644 index 00000000..9039228c --- /dev/null +++ b/api/lib/VideoStore.php @@ -0,0 +1,234 @@ +queue = new VideoQueue("{$root}/queue"); + } + + protected static function filePattern(): string + { + return '[a-f0-9]{64}\.mp4'; + } + + public function queue(): VideoQueue + { + return $this->queue; + } + + /** + * Takes an upload: a file already converted answers as it is, anything else waits in the + * queue, a failed one is tried again. + * + * @return array + * @throws InvalidVideoException when the bytes cannot be a video + */ + public function store(string $bytes, string $originalName, string $folder): array + { + $in = fopen('php://memory', 'r+b'); + fwrite($in, $bytes); + rewind($in); + try { + return $this->storeStream($in, strlen($bytes), $originalName, $folder); + } finally { + fclose($in); + } + } + + /** + * Like store(), straight from the request: the bytes go to disk as they come, and only their + * first ones and their hash are looked at, so a video of 2 GB needs no 2 GB of memory. + * + * @param resource $in + * @return array + * @throws VideoTooLargeException when the stream has more than `$maxBytes` + * @throws InvalidVideoException when the bytes cannot be a video + */ + public function storeStream($in, int $maxBytes, string $originalName, string $folder): array + { + $upload = $this->queue->receive($in, $maxBytes); + try { + if (!self::looksLikeVideo((string) file_get_contents($upload, false, null, 0, 256))) { + throw new InvalidVideoException('Not a video file – use ' . self::SIGNATURE_HINT); + } + if (!preg_match('/^' . UserStore::FOLDER_PATTERN . '$/', $folder)) { + throw new InvalidArgumentException("Invalid folder: {$folder}"); + } + $hash = hash_file('sha256', $upload); + $dir = "{$this->root}/{$folder}"; + if (!is_dir($dir) && !mkdir($dir, 0775, true) && !is_dir($dir)) { + throw new RuntimeException("Cannot create directory {$dir}"); + } + OriginalName::write($dir, "{$hash}.mp4", $originalName); + if (!is_file("{$dir}/{$hash}.mp4")) { + $job = $this->queue->job($folder, $hash); + if ($job === null || $job['state'] === VideoQueue::FAILED) { + $this->queue->add($folder, $hash, $upload, OriginalName::sanitize($originalName)); + } + } + return $this->record("{$folder}/{$hash}.mp4"); + } finally { + if (is_file($upload)) { + unlink($upload); + } + } + } + + /** + * One file as the picker shows it, done or not; null when there is neither file nor job. + * + * @return array|null + */ + public function record(string $file): ?array + { + if (!self::isValidName($file)) { + return null; + } + if (is_file("{$this->root}/{$file}")) { + $folder = self::folderOf($file); + $dir = $folder === '' ? $this->root : "{$this->root}/{$folder}"; + return [ + 'file' => $file, + 'name' => OriginalName::read($dir, basename($file)), + ...$this->details("{$this->root}/{$file}"), + 'bytes' => (int) filesize("{$this->root}/{$file}"), + ]; + } + $job = $this->jobOf($file); + return $job === null ? null : $this->pending($job); + } + + /** A video is there for deleting while it is converted or failed, too. */ + public function exists(string $file): bool + { + return parent::exists($file) || $this->jobOf($file) !== null; + } + + /** Lists the converted files like every store does, and the ones on their way after them. */ + public function list(string $folder, bool $seesAll, array $owners): array + { + $items = parent::list($folder, $seesAll, $owners); + foreach ($this->queue->jobs() as $job) { + if ($seesAll || $job['folder'] === $folder) { + $items[] = $this->pending($job) + [ + 'updatedAt' => (string) $job['createdAt'], + 'owner' => $owners[$job['folder']] ?? '', + 'deletable' => true, + ]; + } + } + return $items; + } + + public function delete(string $file): bool + { + if (!self::isValidName($file)) { + return false; + } + $folder = self::folderOf($file); + $hash = basename($file, '.mp4'); + $removed = $folder !== '' && $this->queue->remove($folder, $hash); + if (is_file("{$this->root}/{$file}")) { + $removed = parent::delete($file) || $removed; + $dir = $folder === '' ? $this->root : "{$this->root}/{$folder}"; + foreach (["{$dir}/{$hash}.jpg", "{$dir}/{$hash}.meta.json"] as $path) { + if (is_file($path)) { + unlink($path); + } + } + } elseif ($removed) { + OriginalName::delete("{$this->root}/{$folder}", "{$hash}.mp4"); + } + return $removed; + } + + /** Called by the worker once the MP4 is in place: what the editor reads instead of probing. */ + public static function writeDetails(string $dir, string $hash, array $details): void + { + $json = json_encode($details, JSON_UNESCAPED_SLASHES); + AtomicFile::write("{$dir}/{$hash}.meta.json", (string) $json, AtomicFile::SERVED, 'video details'); + } + + /** @return array */ + protected function details(string $path): array + { + $meta = substr($path, 0, -strlen('.mp4')) . '.meta.json'; + $raw = is_file($meta) ? file_get_contents($meta) : false; + $details = $raw === false ? null : json_decode($raw, true); + $details = is_array($details) ? $details : []; + $file = substr($path, strlen($this->root) + 1); + return [ + 'status' => 'ready', + 'width' => (int) ($details['width'] ?? 0), + 'height' => (int) ($details['height'] ?? 0), + 'duration' => (float) ($details['duration'] ?? 0), + 'fps' => (float) ($details['fps'] ?? 0), + 'audio' => (bool) ($details['audio'] ?? false), + 'poster' => substr($file, 0, -strlen('.mp4')) . '.jpg', + ]; + } + + /** @return array|null */ + private function jobOf(string $file): ?array + { + $folder = self::folderOf($file); + return $folder === '' ? null : $this->queue->job($folder, basename($file, '.mp4')); + } + + /** + * @param array $job + * @return array + */ + private function pending(array $job): array + { + return [ + 'file' => "{$job['folder']}/{$job['hash']}.mp4", + 'name' => (string) $job['name'], + 'status' => (string) $job['state'], + 'progress' => (float) $job['progress'], + 'error' => (string) $job['error'], + 'bytes' => 0, + ]; + } + + private static function looksLikeVideo(string $bytes): bool + { + if (strlen($bytes) < 16) { + return false; + } + return substr($bytes, 4, 4) === 'ftyp' + || str_starts_with($bytes, "\x1A\x45\xDF\xA3") + || (str_starts_with($bytes, 'RIFF') && substr($bytes, 8, 4) === 'AVI ') + || ($bytes[0] === "\x47" && strlen($bytes) > 188 && $bytes[188] === "\x47"); + } +} + +/** Thrown by VideoStore::store when the uploaded bytes cannot be a video. */ +final class InvalidVideoException extends RuntimeException +{ +} + +/** Thrown by VideoQueue::receive when an upload is bigger than the API takes. */ +final class VideoTooLargeException extends RuntimeException +{ +} diff --git a/api/lib/VideoWorker.php b/api/lib/VideoWorker.php new file mode 100644 index 00000000..70d2b0c2 --- /dev/null +++ b/api/lib/VideoWorker.php @@ -0,0 +1,150 @@ +store->queue()->next(); + if ($job === null) { + return false; + } + $this->convert((string) $job['folder'], (string) $job['hash']); + return true; + } + + private function convert(string $folder, string $hash): void + { + $queue = $this->store->queue(); + if (!$queue->update($folder, $hash, ['state' => VideoQueue::CONVERTING, 'progress' => 0, 'error' => ''])) { + return; + } + $dir = "{$this->videos}/{$folder}"; + $tag = bin2hex(random_bytes(4)); + $tmpVideo = "{$dir}/{$hash}.{$tag}.tmp.mp4"; + $tmpPoster = "{$dir}/{$hash}.{$tag}.tmp.jpg"; + try { + $plan = VideoFormat::plan(self::probe($queue->source($folder, $hash))); + $finished = $this->run( + VideoFormat::convertArgs($queue->source($folder, $hash), $tmpVideo, $plan), + $plan['duration'], + static fn(float $p): bool => $queue->update($folder, $hash, ['progress' => round($p, 3)]), + ); + if (!$finished) { + return; + } + $details = VideoFormat::details(self::probe($tmpVideo)); + $this->run(VideoFormat::posterArgs($tmpVideo, $tmpPoster, $details['duration']), 0, null); + if ($queue->job($folder, $hash) === null) { + return; + } + chmod($tmpVideo, AtomicFile::SERVED); + chmod($tmpPoster, AtomicFile::SERVED); + VideoStore::writeDetails($dir, $hash, $details); + rename($tmpPoster, "{$dir}/{$hash}.jpg"); + rename($tmpVideo, "{$dir}/{$hash}.mp4"); + $queue->remove($folder, $hash); + } catch (Throwable $e) { + $queue->dropSource($folder, $hash); + $queue->update($folder, $hash, ['state' => VideoQueue::FAILED, 'error' => self::shorten($e->getMessage())]); + } finally { + foreach ([$tmpVideo, $tmpPoster] as $path) { + if (is_file($path)) { + unlink($path); + } + } + } + } + + /** @return array */ + private static function probe(string $path): array + { + [$code, $out, $err] = self::capture(VideoFormat::probeArgs($path)); + $probe = json_decode($out, true); + if ($code !== 0 || !is_array($probe)) { + throw new RuntimeException('ffmpeg cannot read this file' . ($err === '' ? '' : ": {$err}")); + } + return $probe; + } + + /** + * Runs ffmpeg, reporting progress from its `-progress pipe:1` lines to `$report`. Returns false + * when `$report` says the job is gone, after stopping ffmpeg. + * + * @param list $args + * @param (callable(float):bool)|null $report + */ + private function run(array $args, float $duration, ?callable $report): bool + { + $err = tempnam(sys_get_temp_dir(), 'video-err-'); + $process = proc_open($args, [0 => ['file', '/dev/null', 'r'], 1 => ['pipe', 'w'], 2 => ['file', $err, 'w']], $pipes); + if (!is_resource($process)) { + throw new RuntimeException('Cannot start ffmpeg'); + } + $last = 0.0; + $alive = true; + while (($line = fgets($pipes[1])) !== false) { + $share = $report === null ? null : VideoFormat::progress($line, $duration); + if ($share !== null && microtime(true) - $last >= self::PROGRESS_EVERY) { + $last = microtime(true); + if (!$report($share)) { + $alive = false; + proc_terminate($process); + break; + } + } + } + fclose($pipes[1]); + $code = proc_close($process); + $message = trim((string) file_get_contents($err)); + unlink($err); + if (!$alive) { + return false; + } + if ($code !== 0) { + throw new RuntimeException($message === '' ? "ffmpeg stopped with code {$code}" : $message); + } + return true; + } + + /** + * @param list $args + * @return array{0:int,1:string,2:string} + */ + private static function capture(array $args): array + { + $process = proc_open($args, [0 => ['file', '/dev/null', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + if (!is_resource($process)) { + throw new RuntimeException('Cannot start ffprobe'); + } + $out = (string) stream_get_contents($pipes[1]); + $err = trim((string) stream_get_contents($pipes[2])); + fclose($pipes[1]); + fclose($pipes[2]); + return [proc_close($process), $out, $err]; + } + + /** The last lines of what ffmpeg said: enough to tell why, not a log. */ + private static function shorten(string $message): string + { + $lines = array_values(array_filter(array_map('trim', explode("\n", $message)), static fn(string $l): bool => $l !== '')); + $text = implode(' · ', array_slice($lines, -2)); + return mb_strlen($text) > 300 ? mb_substr($text, 0, 300) . '…' : $text; + } +} diff --git a/src/core/commands/assetCommands.ts b/src/core/commands/assetCommands.ts index 58814888..50caae29 100644 --- a/src/core/commands/assetCommands.ts +++ b/src/core/commands/assetCommands.ts @@ -10,6 +10,7 @@ import type { TextureAsset, TilemapLayer, VariablesAsset, + VideoAsset, } from '../model/Asset'; import type { AssetType } from '../model/AssetTypes'; import { embeddedIn } from '../music/embedded'; @@ -33,6 +34,7 @@ import { } from '../model/Scope'; import type { SceneObject } from '../model/SceneObject'; import { isTextureFile } from '../model/texture'; +import { isVideoFile } from '../model/video'; import { cellsWithout } from '../model/tilemap'; import { ASSET_FX_KEY } from '../properties/assetFx'; import { coerceValue, defaultsFor, initialOf, initialPropsFor } from '../properties/PropertySchema'; @@ -108,6 +110,8 @@ export function buildAsset(type: AssetType, name: string, id = newId()): Asset { return { ...base, type: 'font', file: '' }; case 'sound': return { ...base, type: 'sound', file: '' }; + case 'video': + return { ...base, type: 'video', file: '', width: 0, height: 0, duration: 0, fps: 0, audio: false }; case 'soundfx': return { ...base, type: 'soundfx', graph: createEmptyGraph() }; case 'instrument': @@ -311,6 +315,25 @@ export class CreateSoundCommand extends ImportFileCommand { } } +/** What a video says about itself once the worker has made it (#402), copied into the asset. */ +export interface VideoDetails { + width: number; + height: number; + duration: number; + fps: number; + audio: boolean; +} + +export class CreateVideoCommand extends ImportFileCommand { + constructor(name: string, file: string, details: VideoDetails) { + super( + { id: newId(), name, props: defaultsFor('video'), type: 'video', file, ...details }, + 'video', + isVideoFile, + ); + } +} + /** * Makes an asset a song kept to itself a project asset (#355): it shows in the lists, and a name a * listed asset has already gets a number. diff --git a/src/core/commands/clipCommands.ts b/src/core/commands/clipCommands.ts index 72dca76f..c851c096 100644 --- a/src/core/commands/clipCommands.ts +++ b/src/core/commands/clipCommands.ts @@ -1,29 +1,41 @@ -/** The clips of a sound (#357): stretches of its file with a name, as frames are of a texture. */ +/** + * The clips of a sound (#357) or a video (#402): stretches of its file with a name, as frames are + * of a texture. + */ import { newId } from '../ids'; -import type { SoundAsset, SoundClip } from '../model/Asset'; +import type { SoundClip } from '../model/Asset'; import type { PropertyBag, PropertyValue } from '../model/Properties'; import type { Project } from '../model/Project'; -import { clipsOf, nextClipName, uniqueClipName } from '../model/sound'; +import { type ClipOwner, clipsOf, hasClips, nextClipName, uniqueClipName } from '../model/sound'; +import type { PropertyDef } from '../properties/PropertySchema'; import { CLIP_ROWS } from '../properties/sounds'; +import { VIDEO_CLIP_ROWS } from '../properties/videos'; import { type Removed, removeFirst, restoreAt } from '../util/list'; import { type Command, CommandError } from './Command'; import { requireAsset } from './require'; -function requireSound(project: Project, assetId: string): SoundAsset { +/** A clip is the same shape on a sound and on a video; only its rows differ. */ +type Clip = SoundClip; + +function requireSound(project: Project, assetId: string): ClipOwner { const asset = requireAsset(project, assetId); - if (asset.type !== 'sound') throw new CommandError(`${asset.name} is not a sound`); + if (!hasClips(asset)) throw new CommandError(`${asset.name} has no clips`); return asset; } -function requireClip(asset: SoundAsset, clipId: string): SoundClip { - const clip = clipsOf(asset).find((c) => c.id === clipId); +function requireClip(asset: ClipOwner, clipId: string): Clip { + const clip = clipsOf(asset).find((c) => c.id === clipId); if (!clip) throw new CommandError(`Clip ${clipId} not found in ${asset.name}`); return clip; } -/** What a clip may hold: a row of `CLIP_ROWS`, a finite number at least 0, a curve it offers. */ -function heldClipValue(key: string, value: PropertyValue): PropertyValue { - const def = CLIP_ROWS.find((row) => row.key === key); +function clipRowsOf(asset: ClipOwner): readonly PropertyDef[] { + return asset.type === 'video' ? VIDEO_CLIP_ROWS : CLIP_ROWS; +} + +/** What a clip may hold: a row of its asset's clip rows, a finite number at least 0, a curve it offers. */ +function heldClipValue(asset: ClipOwner, key: string, value: PropertyValue): PropertyValue { + const def = clipRowsOf(asset).find((row) => row.key === key); if (!def) throw new CommandError(`A clip has no "${key}"`); if (def.kind === 'number' && (typeof value !== 'number' || !Number.isFinite(value) || value < 0)) throw new CommandError(`${def.label} needs a number of at least 0`); @@ -47,14 +59,14 @@ export class AddClipCommand implements Command { execute(project: Project): void { const asset = requireSound(project, this.assetId); const props: PropertyBag = {}; - for (const [key, value] of Object.entries(this.props)) props[key] = heldClipValue(key, value); + for (const [key, value] of Object.entries(this.props)) props[key] = heldClipValue(asset, key, value); // Named per execution: a redo has to fit the names that exist when it runs. - asset.clips = [...clipsOf(asset), { id: this.clipId, name: nextClipName(asset), props }]; + asset.clips = [...clipsOf(asset), { id: this.clipId, name: nextClipName(asset), props }]; } undo(project: Project): void { const asset = requireSound(project, this.assetId); - asset.clips = clipsOf(asset).filter((c) => c.id !== this.clipId); + asset.clips = clipsOf(asset).filter((c) => c.id !== this.clipId); } } @@ -73,8 +85,11 @@ export class SetClipPropsCommand implements Command { ) {} execute(project: Project): void { - const clip = requireClip(requireSound(project, this.assetId), this.clipId); - const held = Object.entries(this.values).map(([key, value]) => [key, heldClipValue(key, value)] as const); + const asset = requireSound(project, this.assetId); + const clip = requireClip(asset, this.clipId); + const held = Object.entries(this.values).map( + ([key, value]) => [key, heldClipValue(asset, key, value)] as const, + ); this.previous = held.map(([key]) => [key, clip.props[key]]); for (const [key, value] of held) clip.props[key] = value; } @@ -117,7 +132,7 @@ export class RenameClipCommand implements Command { */ export class RemoveClipCommand implements Command { readonly label = 'Delete clip'; - private removed?: Removed; + private removed?: Removed; constructor( private readonly assetId: string, @@ -126,7 +141,7 @@ export class RemoveClipCommand implements Command { execute(project: Project): void { const asset = requireSound(project, this.assetId); - const clips = [...clipsOf(asset)]; + const clips = [...clipsOf(asset)]; const removed = removeFirst(clips, (c) => c.id === this.clipId); if (!removed) throw new CommandError(`Clip ${this.clipId} not found in ${asset.name}`); this.removed = removed; @@ -136,7 +151,7 @@ export class RemoveClipCommand implements Command { undo(project: Project): void { if (!this.removed) return; const asset = requireSound(project, this.assetId); - const clips = [...clipsOf(asset)]; + const clips = [...clipsOf(asset)]; restoreAt(clips, [this.removed]); asset.clips = clips; } diff --git a/src/core/model/Asset.ts b/src/core/model/Asset.ts index 302c071f..1ff654c1 100644 --- a/src/core/model/Asset.ts +++ b/src/core/model/Asset.ts @@ -348,6 +348,31 @@ export interface SoundClip { stream?: SoundStream; } +/** + * `file` names the video in `data/videos/` (#402): `{folder}/{sha256}.mp4`, the MP4 the worker made + * from the upload, with a poster `.jpg` and its details beside it. The size, length, frame rate and + * whether it has sound are copied in when it is picked, so a sprite can be measured before the file + * loads. A sprite wears it through `texture`; its clips are stretches of it, as a sound's are. + */ +export interface VideoAsset extends AssetBase { + type: 'video'; + file: string; + width: number; + height: number; + duration: number; + fps: number; + audio: boolean; + /** Absent for a video without clips. */ + clips?: VideoClip[]; +} + +/** A stretch of a video with a name, `props` its start and end (see `properties/videos.ts`). */ +export interface VideoClip { + id: string; + name: string; + props: PropertyBag; +} + /** * What a sound is played through (#311): the graph runs from its `Input` over effect nodes to its * `Output`, and a `Play sound` naming it sends the voice along that line (see `core/soundfx`). @@ -658,7 +683,8 @@ export type Asset = | PatternAsset | SongAsset | BusAsset - | ParticlesAsset; + | ParticlesAsset + | VideoAsset; /** The assets whose `variables` are declared by the user – see `requireVariableOwner`. */ export type VariableOwnerAsset = BehaviorAsset | VariablesAsset | StructAsset; diff --git a/src/core/model/AssetTypes.ts b/src/core/model/AssetTypes.ts index 6db62ca2..f826bf61 100644 --- a/src/core/model/AssetTypes.ts +++ b/src/core/model/AssetTypes.ts @@ -30,6 +30,7 @@ export const ASSET_TYPES = [ 'song', 'bus', 'particles', + 'video', ] as const; export type AssetType = (typeof ASSET_TYPES)[number]; @@ -66,6 +67,7 @@ export const ASSET_TYPE_LABELS: Record = { song: 'Song', bus: 'Bus', particles: 'Particles', + video: 'Video', }; /** @@ -79,7 +81,8 @@ export const ASSET_TYPE_LABELS: Record = { * A shader function is part of other shaders (#303), worn by nothing itself. A sound FX is what a * sound is played through (#311), heard but never seen, an instrument what a note is played * with (#331), a pattern the notes it plays (#333) and a song what lays them out (#334). A bus is - * where sounds are mixed on their way out (#325). + * where sounds are mixed on their way out (#325). A video is a moving picture a sprite wears through + * `texture` (#402), as it wears an image. */ export type PlaceableAssetType = Exclude< AssetType, @@ -103,6 +106,7 @@ export type PlaceableAssetType = Exclude< | 'bus' | 'variables' | 'struct' + | 'video' >; const NOT_PLACEABLE: ReadonlySet = new Set([ @@ -126,6 +130,7 @@ const NOT_PLACEABLE: ReadonlySet = new Set([ 'bus', 'variables', 'struct', + 'video', ]); export function isPlaceableType(type: AssetType): type is PlaceableAssetType { diff --git a/src/core/model/sound.ts b/src/core/model/sound.ts index ccfc745c..4ae98835 100644 --- a/src/core/model/sound.ts +++ b/src/core/model/sound.ts @@ -1,6 +1,6 @@ import { nextFreeName, nextNumberedName } from '../commands/naming'; import { newId } from '../ids'; -import type { Asset, SoundAsset, SoundClip } from './Asset'; +import type { Asset, SoundAsset, SoundClip, VideoAsset } from './Asset'; import { findAsset, type Project } from './Project'; import { storedFilePattern } from './storedFile'; @@ -13,10 +13,18 @@ export function isSoundFile(name: string): boolean { const CLIP_SEPARATOR = ' / '; const DEFAULT_CLIP_NAME = 'Clip'; -export function clipsOf(asset: SoundAsset): readonly SoundClip[] { +/** The clips of a sound (#357) or a video (#402). */ +export function clipsOf(asset: { clips?: C[] }): readonly C[] { return asset.clips ?? []; } +/** What has clips: a sound and a video, each a file with named stretches of it. */ +export type ClipOwner = SoundAsset | VideoAsset; + +export function hasClips(asset: Asset): asset is ClipOwner { + return asset.type === 'sound' || asset.type === 'video'; +} + /** What a sound field names (#357): a sound, the whole file, or one of its clips. */ export interface SoundRef { asset: SoundAsset; @@ -39,10 +47,12 @@ export function findSound(project: Project, id: string): SoundRef | null { return null; } -/** The asset among `assets` an id names: the asset itself, or the sound one of whose clips it is. */ +/** The asset among `assets` an id names: the asset itself, or the one one of whose clips it is. */ export function assetNamedBy(assets: readonly Asset[], id: string): Asset | undefined { if (id === '') return undefined; - return assets.find((a) => a.id === id || (a.type === 'sound' && clipsOf(a).some((c) => c.id === id))); + return assets.find( + (a) => a.id === id || (hasClips(a) && clipsOf<{ id: string }>(a).some((c) => c.id === id)), + ); } /** @@ -50,8 +60,8 @@ export function assetNamedBy(assets: readonly Asset[], id: string): Asset | unde * each old id with its new one in `ids` so the places that named a clip can follow. */ export function renewClipIds(copy: Asset, ids: Map): void { - if (copy.type !== 'sound') return; - for (const clip of clipsOf(copy)) { + if (!hasClips(copy)) return; + for (const clip of clipsOf<{ id: string }>(copy)) { const id = newId(); ids.set(clip.id, id); clip.id = id; @@ -63,18 +73,18 @@ export function soundRefName(ref: SoundRef, soundName = ref.asset.name): string return ref.clip ? `${soundName}${CLIP_SEPARATOR}${ref.clip.name}` : soundName; } -/** "Clip 1", 2, 3 …: the first number none of the sound's clips has. */ -export function nextClipName(asset: SoundAsset): string { +/** "Clip 1", 2, 3 …: the first number none of the asset's clips has. */ +export function nextClipName(asset: ClipOwner): string { return nextNumberedName( - clipsOf(asset).map((c) => c.name), + clipsOf<{ name: string }>(asset).map((c) => c.name), DEFAULT_CLIP_NAME, ); } -/** A name free among the sound's clips: `name`, else "name 2", "name 3" … */ -export function uniqueClipName(asset: SoundAsset, name: string, ignoreId = ''): string { +/** A name free among the asset's clips: `name`, else "name 2", "name 3" … */ +export function uniqueClipName(asset: ClipOwner, name: string, ignoreId = ''): string { return nextFreeName( - clipsOf(asset) + clipsOf<{ id: string; name: string }>(asset) .filter((c) => c.id !== ignoreId) .map((c) => c.name), name, diff --git a/src/core/model/texture.ts b/src/core/model/texture.ts index eb732d4d..02cf36b0 100644 --- a/src/core/model/texture.ts +++ b/src/core/model/texture.ts @@ -4,6 +4,7 @@ import { findPixelFrame, pixelFrameRect } from './pixelart'; import { type PropertyBag, textOf, type Vec2 } from './Properties'; import { findAsset, type Project } from './Project'; import { storedFilePattern } from './storedFile'; +import { findVideo, isVideoFile, videoKey, videoPlayOf, type VideoRef } from './video'; const FILE_PATTERN = storedFilePattern('png|jpg|webp|gif'); @@ -95,13 +96,17 @@ export function resolveTexture(project: Project, props: PropertyBag): TextureRef return resolvePicture(project, textOf(props.texture ?? '')); } -/** A texture, a frame or a generating asset; any other asset's id is no picture. */ +/** A texture, a frame, a video or a clip of one, or a generating asset; any other asset's id is no picture. */ export function resolvePicture(project: Project, id: string): TextureRef | null { if (!id) return null; const asset = findAsset(project, id); - if (asset) return asset.type === 'texture' ? refFor(asset, null) : generatedRef(asset, project); + if (asset?.type === 'texture') return refFor(asset, null); + if (asset?.type === 'video') return videoRef({ asset, clip: null }); + if (asset) return generatedRef(asset, project); const found = findFrame(project, id); if (found) return refFor(found.asset, found.frame); + const video = findVideo(project, id); + if (video) return videoRef(video); // A frame of pixel art (#130) is a cut of its generated sheet, as a texture frame is of its image. const pixel = findPixelFrame(project, id); return pixel ? generatedRef(pixel.asset, project, pixelFrameRect(pixel.asset, pixel.index)) : null; @@ -128,6 +133,19 @@ function generatedRef(asset: Asset, project: Project, rect: FrameRect | null = n }; } +/** A playing video (#402) reads like a texture of its size; the key says what plays, see `videoKey`. */ +function videoRef(ref: VideoRef): TextureRef | null { + if (!isVideoFile(ref.asset.file)) return null; + return { + file: videoKey(videoPlayOf(ref)), + smooth: true, + rect: null, + sourceWidth: ref.asset.width, + sourceHeight: ref.asset.height, + slice: null, + }; +} + function refFor(asset: TextureAsset, frame: TextureFrame | null): TextureRef | null { if (!isTextureFile(asset.file)) return null; return { diff --git a/src/core/model/video.ts b/src/core/model/video.ts new file mode 100644 index 00000000..0076eaff --- /dev/null +++ b/src/core/model/video.ts @@ -0,0 +1,110 @@ +import { videoLoopsOf, videoSpanOf, videoVolumeOf } from '../properties/videos'; +import type { VideoAsset, VideoClip } from './Asset'; +import { findAsset, type Project } from './Project'; +import { clipsOf } from './sound'; +import { storedFilePattern } from './storedFile'; + +const FILE_PATTERN = storedFilePattern('mp4'); + +export function isVideoFile(name: string): boolean { + return FILE_PATTERN.test(name); +} + +/** The still the worker cut from a video (#402), lying beside it. */ +export function posterFileOf(file: string): string { + return file.replace(/\.mp4$/, '.jpg'); +} + +/** What a texture field names when it names a video: the whole video, or one of its clips. */ +export interface VideoRef { + asset: VideoAsset; + clip: VideoClip | null; +} + +/** A video or a clip of one, by id; clip ids share the generator with asset ids, as frames do. */ +export function findVideo(project: Project, id: string): VideoRef | null { + if (id === '') return null; + const asset = findAsset(project, id); + if (asset) return asset.type === 'video' ? { asset, clip: null } : null; + for (const video of project.assets) { + if (video.type !== 'video') continue; + const clip = clipsOf(video).find((c) => c.id === id); + if (clip) return { asset: video, clip }; + } + return null; +} + +/** Every id that means "this video": the asset itself and each of its clips. */ +export function videoRefIds(asset: VideoAsset): string[] { + return [asset.id, ...clipsOf(asset).map((c) => c.id)]; +} + +/** + * The stretch a clip plays, inside the video: a start before its end, an end of 0 or past the + * video's length being its end. Without a length (not known yet) the asked values stand. + */ +export function videoSpan(duration: number, start: number, end: number): { start: number; end: number } { + const length = duration > 0 ? duration : Infinity; + const from = Math.min(Math.max(0, start), Number.isFinite(length) ? Math.max(0, length - 0.001) : start); + const to = end > 0 ? Math.min(end, length) : length; + return { start: from, end: to > from ? to : length }; +} + +/** How a video is played: which file, which stretch of it, whether it repeats, how loud. */ +export interface VideoPlay { + file: string; + start: number; + /** Seconds; Infinity for the end of the video, wherever that is. */ + end: number; + loop: boolean; + volume: number; +} + +export function videoPlayOf(ref: VideoRef): VideoPlay { + const asked = ref.clip ? videoSpanOf(ref.clip.props) : { start: 0, end: 0 }; + const span = videoSpan(ref.asset.duration, asked.start, asked.end); + return { + file: ref.asset.file, + start: span.start, + end: ref.asset.duration > 0 && span.end >= ref.asset.duration ? Infinity : span.end, + loop: videoLoopsOf(ref.asset.props), + volume: videoVolumeOf(ref.asset.props), + }; +} + +const PREFIX = 'video:'; + +/** + * The name a playing video goes by where a picture is named by `TextureRef.file` (see + * `generated.ts`): `video:#~~~`. No uploaded image looks like it, + * and one key is one player: two sprites wearing the same clip show the same picture. + */ +export function videoKey(play: VideoPlay): string { + const end = Number.isFinite(play.end) ? play.end : 0; + return `${PREFIX}${play.file}#${play.start}~${end}~${play.loop ? 1 : 0}~${play.volume}`; +} + +export function isVideoKey(key: string): boolean { + return parseVideoKey(key) !== null; +} + +export function parseVideoKey(key: string): VideoPlay | null { + if (!key.startsWith(PREFIX)) return null; + const hash = key.indexOf('#'); + if (hash < 0) return null; + const file = key.slice(PREFIX.length, hash); + const parts = key.slice(hash + 1).split('~'); + const [start, end, loop, volume] = parts.map(Number); + if (!isVideoFile(file) || parts.length !== 4 || ![start, end, loop, volume].every(Number.isFinite)) + return null; + return { file, start: start!, end: end! > 0 ? end! : Infinity, loop: loop === 1, volume: volume! }; +} + +/** Where in its file a player is after `elapsed` seconds of playing (#402). */ +export function videoTimeAt(play: Pick, elapsed: number): number { + const length = play.end - play.start; + const run = Math.max(0, elapsed); + if (!Number.isFinite(length) || length <= 0) return play.start + run; + if (!play.loop) return play.start + Math.min(run, length); + return play.start + (run % length); +} diff --git a/src/core/properties/PropertySchema.ts b/src/core/properties/PropertySchema.ts index 87f94de9..599699f4 100644 --- a/src/core/properties/PropertySchema.ts +++ b/src/core/properties/PropertySchema.ts @@ -129,6 +129,7 @@ import { import { DEFAULT_SIGN, DEFAULT_TEMPO, MAX_TEMPO, MIN_TEMPO, SIGN_OPTIONS } from '../music/time'; import { ASSET_FX_KEY } from './assetFx'; import { BUS_FX_KEY, BUS_VOLUME_KEY } from './buses'; +import { VIDEO_ROWS } from './videos'; import { Theme } from '../../ui'; /** @@ -138,7 +139,7 @@ import { Theme } from '../../ui'; * `assetTypes` an asset picker offers every placeable asset; `none` is what it calls an empty pick, * `master` offers the master bus as a choice of its own, `noFx` "None" as one where empty means * "from the asset" (#352); `uploadedOnly` keeps drawn pictures out - * of a texture picker (#186). `renderLayers` makes a layer row pick from the project's render layers + * of a texture picker (#186), `moving` lets videos and their clips in (#402). `renderLayers` makes a layer row pick from the project's render layers * instead of its collision layers (#302), with "Inherit" as the first choice of a single layer. */ type Extras = K extends 'number' @@ -153,7 +154,7 @@ type Extras = K extends 'number' noFx?: boolean; } : K extends 'texture' - ? { uploadedOnly?: boolean } + ? { uploadedOnly?: boolean; moving?: boolean } : K extends 'layer' | 'layers' ? { renderLayers?: boolean } : unknown; @@ -529,7 +530,8 @@ const ASSET_SCHEMAS: Record = { { key: 'height', label: 'Height', kind: 'number', default: 100, min: 1, step: 1 }, // The fill while there is no picture, and a tint multiplier once there is one: white = no tint. { key: 'color', label: 'Color', kind: 'color', default: 0xffffff }, - { key: 'texture', label: 'Texture', kind: 'texture', default: '' }, + // A video or a clip of one plays on the sprite (#402). + { key: 'texture', label: 'Texture', kind: 'texture', default: '', moving: true }, // The shader's own parameters follow this row – see `schemaForAsset`. Empty is no shader at all. { key: SHADER_KEY, label: 'Shader', kind: 'asset', default: '', assetTypes: ['shader'] }, ], @@ -760,6 +762,7 @@ const ASSET_SCHEMAS: Record = { ], // Where sounds are mixed (#325): how loud and through which sound FX. Where it goes on is the // master routing's (#351). + video: [...VIDEO_ROWS], bus: [ { key: BUS_VOLUME_KEY, label: 'Volume', kind: 'number', default: 1, min: 0, max: 1, step: 0.05 }, { key: BUS_FX_KEY, label: 'Sound FX', kind: 'asset', default: '', assetTypes: ['soundfx'] }, diff --git a/src/core/properties/textureRef.ts b/src/core/properties/textureRef.ts index 9a23a211..7bb9bb83 100644 --- a/src/core/properties/textureRef.ts +++ b/src/core/properties/textureRef.ts @@ -1,8 +1,10 @@ -import type { Asset, PixelFrame, TextureFrame } from '../model/Asset'; +import type { Asset, PixelFrame, TextureFrame, VideoClip } from '../model/Asset'; import { generatesPictures } from '../model/generated'; import { findPixelFrame, frameLabel } from '../model/pixelart'; import { findAsset, type Project } from '../model/Project'; +import { clipsOf } from '../model/sound'; import { findFrame } from '../model/texture'; +import { findVideo } from '../model/video'; export const NO_TEXTURE_LABEL = 'None'; @@ -14,13 +16,13 @@ export interface TextureChoice { label: string; /** The texture, or the asset that draws its own picture (#185). */ asset: Asset; - /** Null for the whole-picture entry. */ - frame: TextureFrame | PixelFrame | null; + /** Null for the whole-picture entry; a clip for one of a video (#402). */ + frame: TextureFrame | PixelFrame | VideoClip | null; } -/** An asset a texture field may name: an uploaded picture or a drawn one. */ +/** An asset a texture field may name: an uploaded picture, a drawn one, or a video (#402). */ function isPictureAsset(asset: Asset): boolean { - return asset.type === 'texture' || generatesPictures(asset.type); + return asset.type === 'texture' || asset.type === 'video' || generatesPictures(asset.type); } /** @@ -36,7 +38,25 @@ export function textureRefLabel(project: Project | null, id: string): string { const found = findFrame(project, id); if (found) return `${found.asset.name}${FRAME_SEPARATOR}${found.frame.name}`; const pixel = findPixelFrame(project, id); - return pixel ? `${pixel.asset.name}${FRAME_SEPARATOR}${frameLabel(pixel.index)}` : `${id}?`; + if (pixel) return `${pixel.asset.name}${FRAME_SEPARATOR}${frameLabel(pixel.index)}`; + const clip = findVideo(project, id); + return clip?.clip ? `${clip.asset.name}${FRAME_SEPARATOR}${clip.clip.name}` : `${id}?`; +} + +/** The videos and their clips (#402), what a texture field that plays them offers after the pictures. */ +export function videoChoices(project: Project): TextureChoice[] { + const videos = project.assets + .filter((a) => a.type === 'video') + .sort((a, b) => a.name.localeCompare(b.name)); + return videos.flatMap((asset) => [ + { id: asset.id, label: asset.name, asset, frame: null }, + ...clipsOf(asset).map((clip) => ({ + id: clip.id, + label: `${asset.name}${FRAME_SEPARATOR}${clip.name}`, + asset, + frame: clip, + })), + ]); } /** @@ -45,7 +65,9 @@ export function textureRefLabel(project: Project | null, id: string): string { * not depend on upload time; frames keep creation order, which for a sliced sheet is reading order. */ export function pictureChoices(project: Project): TextureChoice[] { - const pictures = project.assets.filter(isPictureAsset).sort((a, b) => a.name.localeCompare(b.name)); + const pictures = project.assets + .filter((asset) => asset.type !== 'video' && isPictureAsset(asset)) + .sort((a, b) => a.name.localeCompare(b.name)); const choices: TextureChoice[] = []; for (const asset of pictures) { choices.push({ id: asset.id, label: asset.name, asset, frame: null }); diff --git a/src/core/properties/videos.ts b/src/core/properties/videos.ts new file mode 100644 index 00000000..ac0d7d38 --- /dev/null +++ b/src/core/properties/videos.ts @@ -0,0 +1,36 @@ +import { booleanOf, type PropertyBag } from '../model/Properties'; +import { coerceAs } from './kinds'; +import type { PropertyDef } from './PropertySchema'; + +/** How loud the video's own sound is, 0–1 (#402). */ +export const VIDEO_VOLUME_KEY = 'volume'; +/** Whether it starts over at its end – a background loop is what a placed video mostly is. */ +export const VIDEO_LOOP_KEY = 'loop'; +/** Where a clip starts and ends, in seconds; an end of 0 is the end of the video. */ +export const VIDEO_START_KEY = 'start'; +export const VIDEO_END_KEY = 'end'; + +export const VIDEO_ROWS: readonly PropertyDef[] = [ + { key: VIDEO_VOLUME_KEY, label: 'Volume', kind: 'number', default: 1, min: 0, max: 1, step: 0.05 }, + { key: VIDEO_LOOP_KEY, label: 'Loop', kind: 'boolean', default: true }, +]; + +/** The rows of a clip of a video, set in the Video tab. */ +export const VIDEO_CLIP_ROWS: readonly PropertyDef[] = [ + { key: VIDEO_START_KEY, label: 'Start (s)', kind: 'number', default: 0, min: 0, step: 0.01, decimals: 3 }, + { key: VIDEO_END_KEY, label: 'End (s)', kind: 'number', default: 0, min: 0, step: 0.01, decimals: 3 }, +]; + +export function videoVolumeOf(bag: PropertyBag): number { + return Math.min(1, Math.max(0, coerceAs(bag[VIDEO_VOLUME_KEY], 'number', 1))); +} + +export function videoLoopsOf(bag: PropertyBag): boolean { + return bag[VIDEO_LOOP_KEY] === undefined ? true : booleanOf(bag[VIDEO_LOOP_KEY]); +} + +/** Start and end of a clip in seconds, `end` 0 for the end of the video. */ +export function videoSpanOf(bag: PropertyBag): { start: number; end: number } { + const seconds = (key: string) => Math.max(0, coerceAs(bag[key], 'number', 0)); + return { start: seconds(VIDEO_START_KEY), end: seconds(VIDEO_END_KEY) }; +} diff --git a/src/core/store/HttpFileStore.ts b/src/core/store/HttpFileStore.ts index d4ac072f..371af9bc 100644 --- a/src/core/store/HttpFileStore.ts +++ b/src/core/store/HttpFileStore.ts @@ -2,10 +2,11 @@ import type { FileStore, StoredFile, UploadedFile } from './FileStore'; import { apiRequest } from './http'; /** - * The API's file collections, one route each; textures, fonts and sounds differ in nothing else. + * The API's file collections, one route each; textures, fonts, sounds and videos differ in nothing + * else. * Pixels (#130) are only ever uploaded – a cel is found by the name in the project, never listed. */ -export type FileRoute = 'textures' | 'fonts' | 'sounds' | 'pixels'; +export type FileRoute = 'textures' | 'fonts' | 'sounds' | 'videos' | 'pixels'; export class HttpFileStore< U extends UploadedFile = UploadedFile, diff --git a/src/core/store/VideoStore.ts b/src/core/store/VideoStore.ts new file mode 100644 index 00000000..11d6bf50 --- /dev/null +++ b/src/core/store/VideoStore.ts @@ -0,0 +1,28 @@ +import type { FileStore, StoredFile, UploadedFile } from './FileStore'; + +/** + * Where an upload is on its way to `data/videos/` (#402): the worker converts one after the other, + * so a video waits, is converted, and is ready or failed. + */ +export type VideoStatus = 'queued' | 'converting' | 'ready' | 'failed'; + +/** + * What the server says about a video. `file` is known the moment the upload is in, long before the + * MP4 exists; the details and the poster come with `ready`, `progress` (0–1) with `converting` + * and `error` with `failed`. + */ +export interface UploadedVideo extends UploadedFile { + status: VideoStatus; + progress?: number; + error?: string; + width?: number; + height?: number; + duration?: number; + fps?: number; + audio?: boolean; + /** `{folder}/{sha256}.jpg`, beside the MP4. */ + poster?: string; +} + +export type StoredVideo = StoredFile & UploadedVideo; +export type VideoStore = FileStore; diff --git a/src/core/store/serialize.ts b/src/core/store/serialize.ts index 24221785..f625756a 100644 --- a/src/core/store/serialize.ts +++ b/src/core/store/serialize.ts @@ -69,6 +69,7 @@ import { decodeCells, encodeCells } from './tilemapCells'; import { isAssetType } from '../model/AssetTypes'; import { isFontFile } from '../model/font'; import { isSoundFile } from '../model/sound'; +import { isVideoFile } from '../model/video'; import { emptyEdit, type InstanceEdit, isEmptyEdit } from '../model/instance'; import { type PropertyBag, type PropertyValue, tagOf, type Vec2 } from '../model/Properties'; import { PROJECT_FORMAT_VERSION, type Project } from '../model/Project'; @@ -117,6 +118,7 @@ import { } from '../properties/PropertySchema'; import { FIELD_SHADER_KEY, particleParamDefs } from '../properties/particles'; import { CLIP_ROWS } from '../properties/sounds'; +import { VIDEO_CLIP_ROWS } from '../properties/videos'; import { SHADER_KEY, shaderParamDefs } from '../properties/shaders'; /** @@ -274,6 +276,23 @@ export function deserializeProject(raw: unknown, fallbackId = ''): Project { ...(clips.length > 0 && { clips }), ...(stream && { stream }), }); + } else if (type === 'video') { + const clips = normalizeClips(a.clips, takenIds, VIDEO_CLIP_ROWS); + const size = (value: unknown) => + typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0; + assets.push({ + id, + type, + name, + props, + file: typeof a.file === 'string' && isVideoFile(a.file) ? a.file : '', + width: Math.round(size(a.width)), + height: Math.round(size(a.height)), + duration: size(a.duration), + fps: size(a.fps), + audio: a.audio === true, + ...(clips.length > 0 && { clips: clips.map(({ id, name, props }) => ({ id, name, props })) }), + }); } else if (type === 'tile') { // A picture naming nothing is kept, like an animation's dangling root: it reads as missing // in the tab and comes back when the texture it named is restored by an undo. @@ -769,8 +788,15 @@ function normalizeTexture( return { file, width, height, frames }; } -/** The clips of a sound (#357); like a frame's, a clip's id may not repeat one the document has. */ -function normalizeClips(raw: unknown, takenIds: Set): SoundClip[] { +/** + * The clips of a sound (#357) or a video (#402); like a frame's, a clip's id may not repeat one the + * document has. + */ +function normalizeClips( + raw: unknown, + takenIds: Set, + rows: readonly PropertyDef[] = CLIP_ROWS, +): SoundClip[] { const clips: SoundClip[] = []; for (const item of Array.isArray(raw) ? raw : []) { const c = asRecord(item); @@ -781,7 +807,7 @@ function normalizeClips(raw: unknown, takenIds: Set): SoundClip[] { clips.push({ id, name: typeof c.name === 'string' ? c.name : '', - props: normalizeBag(c.props, CLIP_ROWS, true), + props: normalizeBag(c.props, rows, true), ...(stream && { stream }), }); } diff --git a/src/editor/App.ts b/src/editor/App.ts index 16d09ad2..e1777293 100644 --- a/src/editor/App.ts +++ b/src/editor/App.ts @@ -7,6 +7,7 @@ import { HttpReleaseStore } from '../core/store/HttpReleaseStore'; import { HttpSnapshotStore } from '../core/store/HttpSnapshotStore'; import { HttpUsersStore } from '../core/store/HttpUsersStore'; import type { StoredTexture, UploadedTexture } from '../core/store/TextureStore'; +import type { StoredVideo, UploadedVideo } from '../core/store/VideoStore'; import { loadPhysics } from '../runtime/physics/engine'; import { projectPictureFactory } from '../viewport/generatedPictures'; import { preloadProjectFiles, setUpProjectFiles } from '../viewport/projectFiles'; @@ -92,6 +93,7 @@ export async function startEditor(app: Application, root: VStack): Promise new HttpFileStore(API_ENDPOINT, 'textures'), new HttpFileStore(API_ENDPOINT, 'fonts'), new HttpFileStore(API_ENDPOINT, 'sounds'), + new HttpFileStore(API_ENDPOINT, 'videos'), new HttpSnapshotStore(API_ENDPOINT), new HttpReleaseStore(API_ENDPOINT), { diff --git a/src/editor/Editor.ts b/src/editor/Editor.ts index ab39c25d..9e460d80 100644 --- a/src/editor/Editor.ts +++ b/src/editor/Editor.ts @@ -9,6 +9,7 @@ import { pluginsOf } from '../core/store/projectPlugins'; import type { FontStore } from '../core/store/FontStore'; import type { SnapshotStore } from '../core/store/SnapshotStore'; import type { SoundStore } from '../core/store/SoundStore'; +import type { VideoStore } from '../core/store/VideoStore'; import { StreamKeeper } from './sound/StreamKeeper'; import type { TextureStore } from '../core/store/TextureStore'; import type { UsersStore } from '../core/store/UsersStore'; @@ -91,6 +92,7 @@ import { PixelArtEditor } from './pixelart/PixelArtEditor'; import { VectorArtEditor } from './vectorart/VectorArtEditor'; import { CurveEditor } from './curve/CurveEditor'; import { SoundEditor } from './sound/SoundEditor'; +import { VideoEditor } from './video/VideoEditor'; import { TextureEditor } from './texture/TextureEditor'; import { installFileImport } from './fileImport'; import type { PluginManager } from './plugins/loader'; @@ -114,6 +116,7 @@ export type CentreTab = | 'pattern' | 'song' | 'game' + | 'video' | 'texture' | 'vectorart' | 'pixelart' @@ -173,6 +176,7 @@ const PANE_ORDER = [ 'instrument', 'pattern', 'song', + 'video', 'texture', 'vectorart', 'pixelart', @@ -201,6 +205,7 @@ const TAB_LABELS: Record = { instrument: 'Instrument', pattern: 'Pattern', song: 'Song', + video: 'Video', texture: 'Texture', vectorart: 'Vector art', pixelart: 'Pixel art', @@ -265,6 +270,7 @@ export class Editor implements PlayHost { readonly shaderEditor: ShaderEditor; readonly particlesEditor: ParticlesEditor; readonly soundEditor: SoundEditor; + readonly videoEditor: VideoEditor; /** Makes the copies streamed sounds play from (#349). */ readonly streamKeeper: StreamKeeper; readonly soundFxEditor: SoundFxEditor; @@ -360,6 +366,7 @@ export class Editor implements PlayHost { textures: TextureStore, fonts: FontStore, sounds: SoundStore, + videos: VideoStore, snapshots: SnapshotStore, releases: ReleaseStore, account: EditorAccount, @@ -373,6 +380,7 @@ export class Editor implements PlayHost { this.particlesEditor = new ParticlesEditor(state); this.streamKeeper = new StreamKeeper(state, sounds); this.soundEditor = new SoundEditor(state, this.streamKeeper); + this.videoEditor = new VideoEditor(state); this.soundFxEditor = new SoundFxEditor(state); this.instrumentEditor = new InstrumentEditor(state); this.patternEditor = new PatternEditor(state, sounds); @@ -449,9 +457,9 @@ export class Editor implements PlayHost { this.workspace = new HStack(); this.leftColumn = new VStack(); - this.assets = new AssetPanel(state, textures, fonts, sounds); + this.assets = new AssetPanel(state, textures, fonts, sounds, videos); this.leftColumn.add(this.assets, this.assetsShare); - this.cleanups.push(installFileImport(app.canvas, { state, textures, fonts, sounds })); + this.cleanups.push(installFileImport(app.canvas, { state, textures, fonts, sounds, videos })); this.leftColumn.add(this.columnSplitter); this.leftColumn.add(this.hierarchy, 1 - this.assetsShare); // A share rather than a height, so a taller or shorter window keeps the split the user chose. @@ -493,6 +501,7 @@ export class Editor implements PlayHost { shader: this.shaderEditor, particles: this.particlesEditor, sound: this.soundEditor, + video: this.videoEditor, soundfx: this.soundFxEditor, instrument: this.instrumentEditor, pattern: this.patternEditor, @@ -1340,6 +1349,7 @@ export class Editor implements PlayHost { if (this.shows('song')) this.songEditor.tick(); else this.songEditor.hideMeters(); if (this.shows('sound')) this.soundEditor.tick(); + if (this.shows('video')) this.videoEditor.tick(); // The tabs of an EQ or a Compressor and their nodes (#380, #381) listen to the same taps: what // is not seen lets go of them first, so it cannot take them from what is. const faces = [ @@ -1392,6 +1402,7 @@ export class Editor implements PlayHost { (this.shows('pattern') && this.patternEditor.isPlaying) || (this.shows('song') && this.songEditor.isPlaying) || (this.shows('sound') && this.soundEditor.isPlaying) || + (this.shows('video') && this.videoEditor.isPlaying) || (this.shows('eq') && this.eqEditor.hasSpectrum) || (this.shows('filter') && this.filterEditor.hasSpectrum) || (this.shows('compressor') && this.compressorEditor.moving) || diff --git a/src/editor/PlaySession.ts b/src/editor/PlaySession.ts index 0ff2684d..e765a235 100644 --- a/src/editor/PlaySession.ts +++ b/src/editor/PlaySession.ts @@ -16,6 +16,7 @@ import { Input } from '../runtime/systems/Input'; import type { World } from '../runtime/World'; import { ui } from '../ui'; import { soundLibrary } from '../viewport/SoundLibrary'; +import { videoLibrary } from '../viewport/VideoLibrary'; import { FrameMeter, type FrameTimes } from './frameMeter'; import { saveArea } from './saves'; import { PanelClock, panelPolicy, type PanelUpdate } from './playThrottle'; @@ -108,6 +109,7 @@ export class PlaySession { // A held note is part of the picture a pause freezes: the sounds stop where they are and carry // on mid-note, rather than playing to their end over a still image. soundLibrary.setSuspended(this.state.paused); + videoLibrary.setPaused(this.state.paused); ui().focus.blur(); this.state.setNotice(this.playNotice(), 'quiet'); } @@ -162,6 +164,7 @@ export class PlaySession { saves: saveArea(runtime.id, 'play'), }; this.run = GameRun.start(runtime, scene.id, this.input, services); + videoLibrary.startRun(); this.updates = { structure: 0, values: 0 }; this.meter.reset(); this.timeSystems(this.run); @@ -212,6 +215,7 @@ export class PlaySession { // And its sounds end with it, looping music included – nothing may outlive the world. soundLibrary.setSuspended(false); soundLibrary.stopAll(); + videoLibrary.endRun(); const world = this.world; this.run = null; this.input.detach(); diff --git a/src/editor/actions.ts b/src/editor/actions.ts index 9be3e9cb..873b8492 100644 --- a/src/editor/actions.ts +++ b/src/editor/actions.ts @@ -32,6 +32,7 @@ import { ownerSong } from '../core/music/embedded'; import type { SceneObject } from '../core/model/SceneObject'; import { pixelArtRefIds } from '../core/model/pixelart'; import { textureRefIds } from '../core/model/texture'; +import { videoRefIds } from '../core/model/video'; import { animationsUsing, cellsPainted, @@ -252,10 +253,12 @@ export function confirmDeleteAsset(state: EditorState, assetId: string): void { ? textureRefIds(asset) : asset.type === 'pixelart' ? pixelArtRefIds(asset) - : [assetId]; + : asset.type === 'video' + ? videoRefIds(asset) + : [assetId]; const references = findPropertyUsages(state.project, ids).length; // An asset that draws its own picture (#185) is worn the same way, only without frames. - const picture = asset.type === 'texture' || generatesPictures(asset.type); + const picture = asset.type === 'texture' || asset.type === 'video' || generatesPictures(asset.type); const animations = picture ? animationsUsing(state.project, ids).length : 0; const tiles = picture ? tilesUsing(state.project, ids).length : 0; const eased = asset.type === 'curve' ? curveUsages(state.project, assetId) : { keys: 0, nodes: 0 }; diff --git a/src/editor/assetImport.ts b/src/editor/assetImport.ts index f019227d..16b215b1 100644 --- a/src/editor/assetImport.ts +++ b/src/editor/assetImport.ts @@ -147,7 +147,7 @@ export function assetNameForFile(fileName: string, fallback: string): string { } /** Opens the browser's file dialog; resolves empty when the dialog was cancelled. */ -function pickFiles(accept: string): Promise { +export function pickFiles(accept: string): Promise { return new Promise((resolve) => { const input = document.createElement('input'); input.type = 'file'; diff --git a/src/editor/assetTabs.ts b/src/editor/assetTabs.ts index b74f13da..8ac38613 100644 --- a/src/editor/assetTabs.ts +++ b/src/editor/assetTabs.ts @@ -10,6 +10,7 @@ export const TAB_OF_ASSET: Partial> = { shaderfunction: 'shader', particles: 'particles', sound: 'sound', + video: 'video', soundfx: 'soundfx', instrument: 'instrument', pattern: 'pattern', diff --git a/src/editor/debug/videoHooks.ts b/src/editor/debug/videoHooks.ts new file mode 100644 index 00000000..42f49765 --- /dev/null +++ b/src/editor/debug/videoHooks.ts @@ -0,0 +1,40 @@ +import type { VideoClip } from '../../core/model/Asset'; +import { findAssetOf } from '../../core/model/Project'; +import { clipsOf } from '../../core/model/sound'; +import { videoLibrary } from '../../viewport/VideoLibrary'; +import { currentVideoPicker } from '../video/VideoPicker'; +import { centreOf, type HookContext } from './hookHelpers'; + +/** The Video tab, its picker and the players behind the sprites (#402). */ +export function videoHooks({ state, editor }: HookContext) { + const { videoEditor } = editor; + return { + videoTab: { + current: () => videoEditor.currentAssetId, + info: () => videoEditor.shownInfo, + rect: () => videoEditor.canvasRect(), + span: () => videoEditor.canvas.shownSpan(), + playhead: () => videoEditor.canvas.playhead, + playing: () => videoEditor.isPlaying, + playButton: () => ({ ...centreOf(videoEditor.playButton), text: videoEditor.playButton.text }), + preview: () => videoEditor.canvas.previewInfo(), + clips: () => { + const asset = findAssetOf(state.project, videoEditor.currentAssetId, 'video'); + return asset + ? clipsOf(asset).map((c) => ({ id: c.id, name: c.name, props: { ...c.props } })) + : []; + }, + pickedClip: () => videoEditor.canvas.pickedClip()?.id ?? null, + /** The middle of the clip strip, and of the track under it, at a second of the video. */ + stripAt: (seconds: number) => videoEditor.canvas.stripAt(seconds), + trackAt: (seconds: number) => videoEditor.canvas.trackAt(seconds), + }, + videoPicker: () => { + const picker = currentVideoPicker(); + return picker ? { ...picker.debugInfo(), waiting: picker.waitingFor } : null; + }, + videoPickerRow: (file: string) => currentVideoPicker()?.rowGlobal(file) ?? null, + /** What each player behind a sprite is doing: its time, whether it runs and is heard. */ + videos: () => videoLibrary.debugInfo(), + }; +} diff --git a/src/editor/debugHook.ts b/src/editor/debugHook.ts index b58b657f..917e9c05 100644 --- a/src/editor/debugHook.ts +++ b/src/editor/debugHook.ts @@ -11,6 +11,7 @@ import { compressorMakeup } from '../core/soundfx/dynamics'; import { bakeHooks } from './debug/bakeHooks'; import { curveHooks } from './debug/curveHooks'; import { soundHooks } from './debug/soundHooks'; +import { videoHooks } from './debug/videoHooks'; import { patternHooks } from './debug/patternHooks'; import { routingHooks } from './debug/routingHooks'; import { songHooks } from './debug/songHooks'; @@ -49,6 +50,7 @@ export function installDebugHook(app: Application, state: EditorState, editor: E ...songHooks(context), ...routingHooks(context), ...soundHooks(context), + ...videoHooks(context), ...bakeHooks(context), ...renderHooks(context), /** diff --git a/src/editor/fileImport.ts b/src/editor/fileImport.ts index 4e1a77fc..ec959eec 100644 --- a/src/editor/fileImport.ts +++ b/src/editor/fileImport.ts @@ -1,14 +1,16 @@ import { type FontImportDeps, importFonts, isFontFileName } from './fontImport'; import { importSounds, isSoundFileName, type SoundImportDeps } from './soundImport'; import { importImages, type TextureImportDeps } from './textureImport'; +import { openVideoPicker } from './video/VideoPicker'; +import { isVideoFileName, uploadVideos, type VideoImportDeps } from './videoImport'; -export type FileImportDeps = TextureImportDeps & FontImportDeps & SoundImportDeps; +export type FileImportDeps = TextureImportDeps & FontImportDeps & SoundImportDeps & VideoImportDeps; /** * Wires drop-onto-`target` and paste, and returns the remover for both. Paste is bound to the * window because the canvas cannot hold focus, and skipped while a DOM text field has it. * - * Pictures, typefaces and sounds arrive the same way, so one pair of handlers sorts them apart. + * Pictures, typefaces, sounds and videos arrive the same way, so one pair of handlers sorts them apart. */ export function installFileImport(target: HTMLElement, deps: FileImportDeps): () => void { const onDragOver = (event: DragEvent) => { @@ -22,7 +24,7 @@ export function installFileImport(target: HTMLElement, deps: FileImportDeps): () if (files.none) { if (!event.dataTransfer?.files.length) return; event.preventDefault(); - deps.state.setNotice('Only image, font and sound files can be dropped into the editor.'); + deps.state.setNotice('Only image, font, sound and video files can be dropped into the editor.'); return; } event.preventDefault(); @@ -50,6 +52,7 @@ interface SortedFiles { images: File[]; fonts: File[]; sounds: File[]; + videos: File[]; none: boolean; } @@ -58,7 +61,14 @@ function sortFiles(data: DataTransfer | null): SortedFiles { const images = all.filter((file) => file.type.startsWith('image/')); const fonts = all.filter((file) => isFontFileName(file.name)); const sounds = all.filter((file) => isSoundFileName(file.name)); - return { images, fonts, sounds, none: images.length + fonts.length + sounds.length === 0 }; + const videos = all.filter((file) => isVideoFileName(file.name)); + return { + images, + fonts, + sounds, + videos, + none: images.length + fonts.length + sounds.length + videos.length === 0, + }; } /** One kind after the other, so a mixed drop ends on one notice instead of three racing ones. */ @@ -67,6 +77,9 @@ function run(files: SortedFiles, deps: FileImportDeps): void { if (files.images.length) await importImages(files.images, deps); if (files.fonts.length) await importFonts(files.fonts, deps); if (files.sounds.length) await importSounds(files.sounds, deps); + // A video is converted first (#402): the picker opens to show how far, and adopts it when done. + if (files.videos.length) + await uploadVideos(files.videos, deps, (pending) => openVideoPicker(deps, pending)); }; void work().catch((error: unknown) => deps.state.setNotice(String(error), 'error')); } diff --git a/src/editor/inspector/SpritePreview.ts b/src/editor/inspector/SpritePreview.ts index 8deedcce..2f6ff2fd 100644 --- a/src/editor/inspector/SpritePreview.ts +++ b/src/editor/inspector/SpritePreview.ts @@ -1,4 +1,5 @@ import { Graphics } from 'pixi.js'; +import { isVideoKey } from '../../core/model/video'; import { SHEET_EDGE } from '../../core/shader/nodes'; import { crisp, hairline, type Size, Theme, ui, Widget } from '../../ui'; import { cancelFrameWork, onFrame } from '../../ui/render'; @@ -91,7 +92,9 @@ export class SpritePreview extends Widget { shaders, ); this.drawn = { x, y, w, h }; - this.animates = Boolean(worn?.time) && this.onScreen(); + // A video (#402) moves by itself, and its player is let go once nobody asks for its picture. + const video = picture.texture !== null && isVideoKey(picture.texture.file); + this.animates = (Boolean(worn?.time) || video) && this.onScreen(); if (this.animates) onFrame(this.frameKey, () => this.invalidate()); } diff --git a/src/editor/inspector/clipSection.ts b/src/editor/inspector/clipSection.ts index 8e222f55..2eaab7b2 100644 --- a/src/editor/inspector/clipSection.ts +++ b/src/editor/inspector/clipSection.ts @@ -1,8 +1,10 @@ import { RemoveClipCommand, RenameClipCommand, SetClipPropsCommand } from '../../core/commands/clipCommands'; -import type { SoundAsset, SoundClip } from '../../core/model/Asset'; -import { findAssetOf, type Project } from '../../core/model/Project'; -import { clipsOf } from '../../core/model/sound'; +import type { SoundClip } from '../../core/model/Asset'; +import { findAsset, type Project } from '../../core/model/Project'; +import { type ClipOwner, clipsOf, hasClips } from '../../core/model/sound'; +import type { PropertyDef } from '../../core/properties/PropertySchema'; import { CLIP_ROWS } from '../../core/properties/sounds'; +import { VIDEO_CLIP_ROWS } from '../../core/properties/videos'; import { Button } from '../../ui'; import { createFieldRow } from './fieldFactory'; import { addGap, addNameField, addNote, addTitle, type InspectorHost } from './host'; @@ -13,13 +15,17 @@ export function clipSignature(project: Project, assetId: string, clipId: string) return currentClip(project, assetId, clipId) ? 'clip' : 'missing'; } -/** Name, the stretch of the file it is and how it is played, and the way to delete it (#357). */ +/** + * Name, the stretch of the file it is and how it is played, and the way to delete it (#357). A clip + * of a video (#402) is only its stretch. + */ export function buildClipSection(host: InspectorHost, assetId: string, clipId: string): void { - const asset = currentSound(host.state.project, assetId); + const asset = currentOwner(host.state.project, assetId); const clip = asset && currentClip(host.state.project, assetId, clipId); if (!asset || !clip) return; + const video = asset.type === 'video'; - addTitle(host, 'SOUND CLIP'); + addTitle(host, video ? 'VIDEO CLIP' : 'SOUND CLIP'); addNameField(host, clip.name, { onChange: (name) => { // The command rejects a blank name but cannot put the old text back into the field. @@ -27,9 +33,14 @@ export function buildClipSection(host: InspectorHost, assetId: string, clipId: s else host.nameField?.setValue(clip.name); }, }); - addNote(host, `Clip of the sound "${asset.name}". Its volume, loop, bus and sound FX are the sound's.`); + addNote( + host, + video + ? `Clip of the video "${asset.name}". Its volume and loop are the video's.` + : `Clip of the sound "${asset.name}". Its volume, loop, bus and sound FX are the sound's.`, + ); - for (const def of CLIP_ROWS) { + for (const def of rowsOf(asset)) { const handlers = previewingHandlers(host, { target: () => currentClip(host.state.project, assetId, clipId), read: (target) => target.props[def.key], @@ -57,16 +68,23 @@ export function refreshClipSection(host: InspectorHost, assetId: string, clipId: const clip = currentClip(host.state.project, assetId, clipId); if (!clip) return false; host.nameField?.setValue(clip.name); - for (const def of CLIP_ROWS) host.fields.get(def.key)?.setValue(clip.props[def.key] ?? def.default); + const asset = currentOwner(host.state.project, assetId); + for (const def of asset ? rowsOf(asset) : []) + host.fields.get(def.key)?.setValue(clip.props[def.key] ?? def.default); return true; } -/** Looked up inside the sound, so a selection left over from a deleted sound edits nothing. */ -function currentClip(project: Project, assetId: string, clipId: string): SoundClip | undefined { - const asset = currentSound(project, assetId); - return asset && clipsOf(asset).find((c) => c.id === clipId); +function rowsOf(asset: ClipOwner): readonly PropertyDef[] { + return asset.type === 'video' ? VIDEO_CLIP_ROWS : CLIP_ROWS; } -function currentSound(project: Project, assetId: string): SoundAsset | undefined { - return findAssetOf(project, assetId, 'sound'); +/** Looked up inside its asset, so a selection left over from a deleted one edits nothing. */ +function currentClip(project: Project, assetId: string, clipId: string): SoundClip | undefined { + const asset = currentOwner(project, assetId); + return asset && clipsOf(asset).find((c) => c.id === clipId); +} + +function currentOwner(project: Project, assetId: string): ClipOwner | undefined { + const asset = findAsset(project, assetId); + return asset && hasClips(asset) ? asset : undefined; } diff --git a/src/editor/inspector/fieldFactory.ts b/src/editor/inspector/fieldFactory.ts index 58e9b002..d1376296 100644 --- a/src/editor/inspector/fieldFactory.ts +++ b/src/editor/inspector/fieldFactory.ts @@ -34,7 +34,12 @@ import { type StructFieldDef, } from '../../core/properties/PropertySchema'; import { referencedAssetId } from '../../core/properties/references'; -import { NO_TEXTURE_LABEL, pictureChoices, textureRefLabel } from '../../core/properties/textureRef'; +import { + NO_TEXTURE_LABEL, + pictureChoices, + textureRefLabel, + videoChoices, +} from '../../core/properties/textureRef'; import { Button, CHEVRON_SLOT, @@ -418,10 +423,12 @@ const REFERENCES: Record = { texture: { none: NO_TEXTURE_LABEL, label: textureRefLabel, - choices: (project, def) => - pictureChoices(project).filter( + choices: (project, def) => [ + ...pictureChoices(project).filter( (choice) => !(def.kind === 'texture' && def.uploadedOnly) || choice.asset.type === 'texture', ), + ...(def.kind === 'texture' && def.moving ? videoChoices(project) : []), + ], }, statemachine: { none: NO_ASSET_LABEL, diff --git a/src/editor/panels/AssetPanel.ts b/src/editor/panels/AssetPanel.ts index 64bf7ca7..d4ae4fe9 100644 --- a/src/editor/panels/AssetPanel.ts +++ b/src/editor/panels/AssetPanel.ts @@ -24,6 +24,7 @@ import { schemaForAsset } from '../../core/properties/PropertySchema'; import { resolveAssetProps } from '../../core/properties/resolve'; import type { FontStore } from '../../core/store/FontStore'; import type { SoundStore } from '../../core/store/SoundStore'; +import type { VideoStore } from '../../core/store/VideoStore'; import type { TextureStore } from '../../core/store/TextureStore'; import { Button, @@ -46,6 +47,7 @@ import { generalSettings, onGeneralSettings } from '../generalSettings'; import type { EditorState } from '../state'; import { openFontPicker } from '../font/FontPicker'; import { openSoundPicker } from '../sound/SoundPicker'; +import { openVideoPicker } from '../video/VideoPicker'; import { openImagePicker } from '../texture/ImagePicker'; /** Above every menu of new assets, so a click on a type reads as making one (#361). */ @@ -84,6 +86,8 @@ export class AssetPanel extends Panel { private readonly fonts: FontStore | null = null, /** And for sounds and the Sound entry. */ private readonly sounds: SoundStore | null = null, + /** And for videos and the Video entry (#402). */ + private readonly videos: VideoStore | null = null, ) { super({ title: 'Assets', padding: 0, gap: 0 }); this.minWidth = 160; @@ -338,7 +342,7 @@ export class AssetPanel extends Panel { } /** - * A texture is a picture, a font and a sound are files, so their menu entries offer what the + * A texture is a picture, a font, a sound and a video are files, so their menu entries offer what the * server already holds instead of creating an empty asset. Uploading a new one is a button * inside that dialog. */ @@ -346,6 +350,7 @@ export class AssetPanel extends Panel { if (type === 'texture') this.importTextures(); else if (type === 'font') this.importFonts(); else if (type === 'sound') this.importSounds(); + else if (type === 'video') this.importVideos(); else if (type === 'pixelart') this.createPixelArt(); else this.createAsset(type); } @@ -393,6 +398,14 @@ export class AssetPanel extends Panel { openSoundPicker({ state: this.state, sounds: this.sounds }); } + private importVideos(): void { + if (!this.videos) { + this.state.setNotice('Video upload is not available.'); + return; + } + openVideoPicker({ state: this.state, videos: this.videos }); + } + /** Double-click / Enter opens the asset's tab; a scene also becomes the active one. Placing is a drag. */ private activate(id: string): void { activateAsset(this.state, id); @@ -471,6 +484,12 @@ export class AssetPanel extends Panel { if (asset.type === 'pixelart' && asset.frames.length > 1) items.push({ label: 'Create animation from frames', onSelect: () => this.createAnimation(asset) }); } + // A video is worn like a texture (#402), at its own size. + if (asset.type === 'video') + items.push({ + label: 'Create sprite from video', + onSelect: () => this.createAsset('sprite', asset.name, spriteFromTexture(this.state.project, id)), + }); items.push({ label: 'Duplicate', onSelect: () => this.duplicateAsset(id) }); items.push({ label: 'Delete…', danger: true, onSelect: () => confirmDeleteAsset(this.state, id) }); const menu = new Menu(items); diff --git a/src/editor/panels/assetGroups.ts b/src/editor/panels/assetGroups.ts index 6c175fb5..b3044f00 100644 --- a/src/editor/panels/assetGroups.ts +++ b/src/editor/panels/assetGroups.ts @@ -28,6 +28,7 @@ export const ASSET_GROUPS: Record = { animation, @@ -72,6 +73,7 @@ export const ASSET_ICON_SVG: Record = { tilemap, variables, vectorart, + video, }; /** A Variables tab showing one list (#252): a list value is no asset. */ diff --git a/src/editor/panels/icons/video.svg b/src/editor/panels/icons/video.svg new file mode 100644 index 00000000..a97764cb --- /dev/null +++ b/src/editor/panels/icons/video.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/editor/shader/ShaderEditor.ts b/src/editor/shader/ShaderEditor.ts index 00584264..f2b84671 100644 --- a/src/editor/shader/ShaderEditor.ts +++ b/src/editor/shader/ShaderEditor.ts @@ -123,6 +123,7 @@ export class ShaderEditor extends AssetTabEditor { /** Called every frame while the tab is showing. */ tick(): void { + if (this.preview.wearsVideo) this.preview.invalidate(); if (!this.animates) return; this.preview.invalidate(); this.syncControls(); diff --git a/src/editor/shader/ShaderPreview.ts b/src/editor/shader/ShaderPreview.ts index f788bd96..d04ec9be 100644 --- a/src/editor/shader/ShaderPreview.ts +++ b/src/editor/shader/ShaderPreview.ts @@ -1,5 +1,6 @@ import { Container, Graphics, Texture } from 'pixi.js'; import type { ShaderAsset } from '../../core/model/Asset'; +import { isVideoKey } from '../../core/model/video'; import type { PropertyBag } from '../../core/model/Properties'; import { SHADER_KEY } from '../../core/properties/shaders'; import { type EdgeMode, SHEET_EDGE } from '../../core/shader/nodes'; @@ -60,6 +61,12 @@ export class ShaderPreview extends Widget { return this.held ?? Math.max(0, shaderLibrary.now() - this.clockOrigin); } + /** Whether the picture is a video (#402): it moves by itself and must be asked for every frame. */ + get wearsVideo(): boolean { + const ref = this.picture?.texture; + return Boolean(ref && isVideoKey(ref.file)); + } + get holding(): boolean { return this.held !== null; } diff --git a/src/editor/state.ts b/src/editor/state.ts index 7105d8c5..bbc02c82 100644 --- a/src/editor/state.ts +++ b/src/editor/state.ts @@ -21,7 +21,7 @@ import { type Project, startScene, } from '../core/model/Project'; -import { clipsOf } from '../core/model/sound'; +import { clipsOf, hasClips } from '../core/model/sound'; import type { PropertyValue } from '../core/model/Properties'; import { deserializeProject, serializeProject } from '../core/store/serialize'; import { requestRender } from '../ui'; @@ -57,7 +57,7 @@ export type Selection = } /** A frame rectangle inside a texture asset (the Texture tab). */ | { kind: 'frame'; assetId: string; frameId: string } - /** A clip of a sound asset (the Sound tab, #357). */ + /** A clip of a sound asset (the Sound tab, #357) or of a video (the Video tab, #402). */ | { kind: 'clip'; assetId: string; clipId: string } /** * One or more shapes of a vector art asset (the Vector art tab, #129). `shapeId` is the primary @@ -797,10 +797,11 @@ export class EditorState { this.select({ kind: 'asset', assetId: s.assetId }); } if (s.kind === 'clip') { - const asset = findAssetOf(this.project, s.assetId, 'sound'); - // As with a frame: the sound stays open when its clip vanished. + const found = findAsset(this.project, s.assetId); + const asset = found && hasClips(found) ? found : undefined; + // As with a frame: the sound or video stays open when its clip vanished. if (!asset) this.select(null); - else if (!clipsOf(asset).some((c) => c.id === s.clipId)) + else if (!clipsOf<{ id: string }>(asset).some((c) => c.id === s.clipId)) this.select({ kind: 'asset', assetId: s.assetId }); } if (s.kind === 'shape') { @@ -893,7 +894,14 @@ export class EditorState { return findAssetOf(this.project, s.assetId, 'sound')?.id ?? null; } - /** The clip of a sound the selection is about (#357); empty for the sound itself. */ + /** The video the selection refers to (#402): the asset, or one of its clips. */ + get selectedVideoId(): string | null { + const s = this.selection; + if (s?.kind !== 'asset' && s?.kind !== 'clip') return null; + return findAssetOf(this.project, s.assetId, 'video')?.id ?? null; + } + + /** The clip of a sound or video the selection is about (#357); empty for the asset itself. */ get selectedClipId(): string { return this.selection?.kind === 'clip' ? this.selection.clipId : ''; } diff --git a/src/editor/video/VideoCanvas.ts b/src/editor/video/VideoCanvas.ts new file mode 100644 index 00000000..91333b83 --- /dev/null +++ b/src/editor/video/VideoCanvas.ts @@ -0,0 +1,633 @@ +import { + Container, + type FederatedPointerEvent, + type FederatedWheelEvent, + Graphics, + Sprite, + type Text, + Texture, +} from 'pixi.js'; +import { AddClipCommand, SetClipPropsCommand } from '../../core/commands/clipCommands'; +import type { VideoAsset, VideoClip } from '../../core/model/Asset'; +import type { PropertyBag, Vec2 } from '../../core/model/Properties'; +import { findAssetOf } from '../../core/model/Project'; +import { clipsOf } from '../../core/model/sound'; +import { + videoLoopsOf, + videoSpanOf, + videoVolumeOf, + VIDEO_END_KEY, + VIDEO_START_KEY, +} from '../../core/properties/videos'; +import { crispText, type Size, textStyle, Theme, ui, Widget, zoomFactor } from '../../ui'; +import { + clampWave, + type ClipBand, + clipAt, + type CutSpan, + draggedSpan, + edgeAt, + fitWave, + rulerLabel, + rulerStep, + secondsToX, + spanOf, + storedCut, + type WaveView, + xToSeconds, + zoomWave, +} from '../sound/waveView'; +import { openVideo } from '../../viewport/VideoLibrary'; +import type { EditorState } from '../state'; +import { containRect } from '../texture/imageGrid'; + +const RULER_HEIGHT = 24; +/** The strip the clips stand in as bands, as in the Sound tab. */ +const STRIP_HEIGHT = 20; +/** The track under it, where the picked clip's edges are dragged. */ +const TRACK_HEIGHT = 32; +const TIMELINE_HEIGHT = RULER_HEIGHT + STRIP_HEIGHT + TRACK_HEIGHT; +const FIT_MARGIN = 16; +const LABEL_SPACING = 70; +/** A clip is a frame long at least; at 60 fps that is this. */ +const LEAST_SPAN = 1 / 60; +const DRAG_START = 3; +/** Clip edges are stored to the millisecond. */ +const STORED_RATE = 1000; +const PREVIEW_PAD = 8; +const LIT_LABEL = textStyle('small').clone(); +LIT_LABEL.fill = 0x30343c; + +type Gesture = + | { kind: 'pan'; pointerX: number; panX: number } + | { kind: 'seek' } + | { + kind: 'span'; + clipId: string | null; + edge: 'start' | 'end' | null; + from: number; + pointerX: number; + moved: boolean; + span: CutSpan; + picks?: string | null; + }; + +/** The tab's own player: the video shown, with its sound, apart from the ones sprites wear. */ +interface Preview { + file: string; + element: HTMLVideoElement; + /** Made once the video has shown its first frame, see `openVideo`. */ + texture: Texture | null; +} + +/** + * The Video tab's canvas (#402): the video on top, a ruler in seconds under it, the clips as bands + * in a strip, and a track where the picked clip stands between two handles. A click in the strip + * picks a clip, a drag there or in the track spans a new one; dragging a handle moves that edge. + * One drag is one undo step. A click in the ruler shows that moment; the wheel zooms the time, the + * middle button pans, as in the Sound tab. + */ +/** The colour in the middle of the frame a video holds; transparent where it has none to draw. */ +export function middlePixel(element: HTMLVideoElement): number[] | null { + if (element.readyState < 2) return null; + const context = new OffscreenCanvas(8, 8).getContext('2d', { willReadFrequently: true }); + if (!context) return null; + context.drawImage(element, 0, 0, 8, 8); + return [...context.getImageData(4, 4, 1, 1).data]; +} + +export class VideoCanvas extends Widget { + private readonly bg = new Graphics(); + private readonly picture = new Sprite(); + private readonly overlay = new Graphics(); + private readonly labels = new Container(); + private readonly texts: Text[] = []; + private readonly stripLabels = new Container(); + private readonly stripTexts: Text[] = []; + private readonly hint = crispText({ text: '', style: textStyle('muted') }); + private view: WaveView = { panX: FIT_MARGIN, pxPerSecond: 100 }; + private needsFit = true; + private assetId = ''; + private gesture: Gesture | null = null; + private preview: Preview | null = null; + /** The span the player is held to while it plays: the picked clip, else the whole video. */ + private playing: CutSpan | null = null; + onTogglePlay: () => void = () => {}; + /** The player stopped by itself (the end of a clip that does not repeat). */ + onStopped: () => void = () => {}; + + constructor(private readonly state: EditorState) { + super(); + this.focusable = true; + this.labels.eventMode = 'none'; + this.stripLabels.eventMode = 'none'; + this.hint.eventMode = 'none'; + this.picture.eventMode = 'none'; + this.addChild(this.bg, this.picture, this.overlay, this.labels, this.stripLabels, this.hint); + this.on('pointerdown', (e: FederatedPointerEvent) => this.onPointerDown(e)); + this.on('globalpointermove', (e: FederatedPointerEvent) => this.onPointerMove(e)); + this.onPointerRelease(() => this.onPointerUp()); + this.on('wheel', (e: FederatedWheelEvent) => this.onWheel(e)); + state.on('project', () => this.invalidate()); + state.on('selection', () => { + if (!this.isPlaying) this.seekTo(this.shownSpan()?.start ?? 0); + this.invalidate(); + }); + } + + override measure(): Size { + return { w: Math.max(this.minWidth, 360), h: Math.max(this.minHeight, TIMELINE_HEIGHT + 120) }; + } + + get currentAssetId(): string { + return this.assetId; + } + + show(assetId: string): void { + if (this.assetId === assetId) return; + this.stop(); + this.gesture = null; + this.assetId = assetId; + this.needsFit = true; + this.redraw(); + } + + saveView(): WaveView { + return { ...this.view }; + } + + restoreView(view: unknown): void { + const held = view as Partial; + if (typeof held.panX !== 'number' || typeof held.pxPerSecond !== 'number') return; + this.view = { panX: held.panX, pxPerSecond: held.pxPerSecond }; + this.needsFit = false; + this.redraw(); + } + + fit(): void { + const asset = this.asset(); + if (!asset || asset.duration <= 0 || this.w <= 0) { + this.needsFit = true; + return; + } + this.view = fitWave(asset.duration, this.w, FIT_MARGIN); + this.needsFit = false; + this.redraw(); + } + + get isPlaying(): boolean { + return this.playing !== null; + } + + /** Plays the picked clip, else the whole video, from its start, with the video's volume and loop. */ + play(): void { + const asset = this.asset(); + const span = this.shownSpan(); + const element = asset && this.player(asset); + if (!asset || !span || !element) return; + this.playing = span; + element.muted = false; + element.volume = videoVolumeOf(asset.props); + element.currentTime = span.start; + void element.play().catch(() => {}); + this.invalidate(); + } + + stop(): void { + this.playing = null; + this.preview?.element.pause(); + this.invalidate(); + } + + /** Where the tab's player stands, in seconds of the video; null while nothing is shown. */ + get playhead(): number | null { + return this.preview && this.preview.file === this.asset()?.file ? this.preview.element.currentTime : null; + } + + /** Called every frame while the tab is on screen: the playhead follows the player, a clip loops. */ + tick(): void { + const element = this.preview?.element; + const span = this.playing; + const asset = this.asset(); + if (!element || !span || !asset) return; + if (element.currentTime >= span.end - LEAST_SPAN / 2 || element.ended) { + if (videoLoopsOf(asset.props)) { + element.currentTime = span.start; + if (element.paused) void element.play().catch(() => {}); + } else { + this.stop(); + this.onStopped(); + } + } + this.redraw(); + } + + /** The picked clip of the video shown, null for none. */ + pickedClip(): VideoClip | null { + const asset = this.asset(); + const id = this.state.selectedClipId; + return (asset && id && clipsOf(asset).find((c) => c.id === id)) || null; + } + + /** What the tab plays: a drag's span while one runs, else the picked clip, else the whole video. */ + shownSpan(): CutSpan | null { + const asset = this.asset(); + if (!asset || asset.duration <= 0) return null; + if (this.gesture?.kind === 'span' && this.gesture.moved) return this.gesture.span; + const clip = this.pickedClip(); + return clip ? this.clipSpan(clip, asset) : { start: 0, end: asset.duration }; + } + + /** The middle of the strip at a second of the video, on screen (test hook). */ + stripAt(seconds: number): Vec2 { + const origin = this.getGlobalPosition(); + return { + x: origin.x + secondsToX(this.view, seconds), + y: origin.y + this.timelineTop + RULER_HEIGHT + STRIP_HEIGHT / 2, + }; + } + + /** The middle of the track at a second of the video, on screen (test hook). */ + trackAt(seconds: number): Vec2 { + const origin = this.getGlobalPosition(); + return { x: origin.x + secondsToX(this.view, seconds), y: origin.y + this.h - TRACK_HEIGHT / 2 }; + } + + /** + * Where the picture is drawn, whether the player has one, and the colour in the middle of the + * frame the player holds, read from the video itself (test hook). + */ + previewInfo(): { + shown: boolean; + x: number; + y: number; + w: number; + h: number; + time: number; + paused: boolean; + pixel: number[] | null; + } { + const p = this.picture.getGlobalPosition(); + return { + pixel: this.preview ? middlePixel(this.preview.element) : null, + shown: this.picture.visible, + x: p.x, + y: p.y, + w: this.picture.width, + h: this.picture.height, + time: this.preview?.element.currentTime ?? 0, + paused: this.preview?.element.paused ?? true, + }; + } + + override destroy(): void { + this.dropPreview(); + super.destroy({ children: true }); + } + + private get timelineTop(): number { + return Math.max(0, this.h - TIMELINE_HEIGHT); + } + + private asset(): VideoAsset | null { + return (this.assetId && findAssetOf(this.state.project, this.assetId, 'video')) || null; + } + + private clipSpan(clip: VideoClip, asset: VideoAsset): CutSpan { + const { start, end } = videoSpanOf(clip.props); + return spanOf(start, end, asset.duration); + } + + private bands(asset: VideoAsset): ClipBand[] { + const g = this.gesture?.kind === 'span' && this.gesture.moved ? this.gesture : null; + const bands = clipsOf(asset).map((clip) => ({ + id: clip.id, + span: g?.clipId === clip.id ? g.span : this.clipSpan(clip, asset), + })); + return g && g.clipId === null ? [...bands, { id: '', span: g.span }] : bands; + } + + /** The tab's player for the video's file, made anew when the file is another one. */ + private player(asset: VideoAsset): HTMLVideoElement | null { + if (!asset.file) return null; + if (this.preview?.file === asset.file) return this.preview.element; + this.dropPreview(); + const held = this.shownSpan()?.start ?? 0; + const preview: Preview = { + file: asset.file, + texture: null, + element: openVideo(`/game-maker/data/videos/${asset.file}`, held, (texture) => { + if (this.preview !== preview) return texture.destroy(true); + preview.texture = texture; + texture.source.on('update', () => this.invalidate()); + // It played for its first frame; until Play it stands at the moment the tab shows. + if (!this.playing) { + preview.element.pause(); + preview.element.currentTime = held; + } + this.invalidate(); + }), + }; + preview.element.addEventListener('seeked', () => this.invalidate()); + this.preview = preview; + return preview.element; + } + + private dropPreview(): void { + const preview = this.preview; + if (!preview) return; + this.preview = null; + this.playing = null; + this.picture.texture = Texture.EMPTY; + preview.element.pause(); + preview.element.removeAttribute('src'); + preview.element.load(); + preview.texture?.destroy(true); + } + + private seekTo(seconds: number): void { + const asset = this.asset(); + const element = asset && this.player(asset); + if (element) element.currentTime = Math.max(0, Math.min(asset.duration, seconds)); + } + + override onKeyDown(event: KeyboardEvent): boolean { + if (event.code !== 'Space' || event.ctrlKey || event.metaKey || !this.asset()) return false; + this.onTogglePlay(); + return true; + } + + private onPointerDown(e: FederatedPointerEvent): void { + const asset = this.asset(); + if (!asset || asset.duration <= 0) return; + const p = this.toLocal(e.global); + const y = p.y - this.timelineTop; + if (y < 0) return; + e.stopPropagation(); + ui().focus.focus(this); + if (e.button === 1) { + this.gesture = { kind: 'pan', pointerX: p.x, panX: this.view.panX }; + return; + } + if (e.button !== 0) return; + const from = Math.max(0, Math.min(asset.duration, xToSeconds(this.view, p.x))); + if (y < RULER_HEIGHT) { + this.gesture = { kind: 'seek' }; + this.stop(); + this.seekTo(from); + return; + } + const start = { + kind: 'span', + from, + pointerX: p.x, + moved: false, + span: { start: from, end: from }, + } as const; + if (y < RULER_HEIGHT + STRIP_HEIGHT) { + this.gesture = { ...start, clipId: null, edge: null, picks: clipAt(this.view, this.bands(asset), p.x) }; + return; + } + const clip = this.pickedClip(); + const span = clip && this.clipSpan(clip, asset); + const edge = span ? edgeAt(this.view, span, p.x) : null; + const beside = + span && !edge && (p.x < secondsToX(this.view, span.start) || p.x > secondsToX(this.view, span.end)); + this.gesture = { ...start, clipId: clip?.id ?? null, edge, picks: beside ? null : undefined }; + } + + private onPointerMove(e: FederatedPointerEvent): void { + const p = this.toLocal(e.global); + const g = this.gesture; + const asset = this.asset(); + if (!g || !asset) return; + if (g.kind === 'pan') { + this.view = clampWave({ ...this.view, panX: g.panX + p.x - g.pointerX }, asset.duration, this.w); + this.needsFit = false; + this.redraw(); + return; + } + if (g.kind === 'seek') { + this.seekTo(xToSeconds(this.view, p.x)); + return; + } + if (!g.moved && g.edge === null && Math.abs(p.x - g.pointerX) < DRAG_START) return; + g.moved = true; + const clip = this.pickedClip(); + const before = clip && clip.id === g.clipId ? this.clipSpan(clip, asset) : g.span; + g.span = draggedSpan(before, g.edge, g.from, xToSeconds(this.view, p.x), asset.duration, LEAST_SPAN); + this.seekTo(g.edge === 'end' ? g.span.end : g.span.start); + this.redraw(); + } + + private onPointerUp(): void { + const g = this.gesture; + this.gesture = null; + if (g?.kind !== 'span') return this.redraw(); + const asset = this.asset(); + if (!g.moved) { + if (asset && g.picks !== undefined) + this.state.select( + g.picks + ? { kind: 'clip', assetId: asset.id, clipId: g.picks } + : { kind: 'asset', assetId: asset.id }, + ); + return this.redraw(); + } + if (asset) this.commit(asset, g); + this.redraw(); + } + + /** A drag's span as the clip's start and end in one step, or as a new clip, which is then picked. */ + private commit(asset: VideoAsset, g: Extract): void { + const next = storedCut(g.span, asset.duration, STORED_RATE); + if (g.clipId === null) { + const add = new AddClipCommand(asset.id, { [VIDEO_START_KEY]: next.start, [VIDEO_END_KEY]: next.end }); + if (this.state.execute(add)) this.state.select({ kind: 'clip', assetId: asset.id, clipId: add.clipId }); + return; + } + const clip = clipsOf(asset).find((c) => c.id === g.clipId); + if (!clip) return; + const now = videoSpanOf(clip.props); + const values: PropertyBag = {}; + if (next.start !== now.start) values[VIDEO_START_KEY] = next.start; + if (next.end !== now.end) values[VIDEO_END_KEY] = next.end; + if (Object.keys(values).length > 0) + this.state.execute( + new SetClipPropsCommand(asset.id, clip.id, values, g.edge ? 'Move clip edge' : 'Span clip'), + ); + } + + private onWheel(e: FederatedWheelEvent): void { + const asset = this.asset(); + const p = this.toLocal(e.global); + if (!asset || asset.duration <= 0 || p.y < this.timelineTop) return; + this.view = clampWave(zoomWave(this.view, zoomFactor(e.deltaY), p.x), asset.duration, this.w); + this.needsFit = false; + this.redraw(); + e.preventDefault(); + } + + override layout(): void { + if (this.needsFit) this.fit(); + } + + override redraw(): void { + const asset = this.asset(); + if (this.needsFit && asset && asset.duration > 0 && this.w > 0) return this.fit(); + const c = Theme.colors; + const top = this.timelineTop; + this.bg.clear().rect(0, 0, this.w, this.h).fill(0x000000); + this.bg.rect(0, top, this.w, RULER_HEIGHT + STRIP_HEIGHT).fill(c.panelHeader); + this.bg.rect(0, top + RULER_HEIGHT - 1, this.w, 1).fill(c.border); + this.bg.rect(0, top + RULER_HEIGHT + STRIP_HEIGHT - 1, this.w, 1).fill(c.border); + this.bg.rect(0, top + RULER_HEIGHT + STRIP_HEIGHT, this.w, TRACK_HEIGHT).fill(c.viewport); + this.overlay.clear(); + this.hint.visible = !asset?.file; + this.hint.text = !asset ? 'No video selected – pick one in the Assets panel.' : 'Not uploaded.'; + this.hint.position.set(Theme.padding * 2, Theme.padding * 2); + if (!asset?.file) { + this.picture.visible = false; + this.syncLabels([]); + this.syncStrip([]); + return; + } + this.drawPicture(asset, top); + const left = Math.max(0, secondsToX(this.view, 0)); + const right = Math.min(this.w, secondsToX(this.view, asset.duration)); + this.drawRuler(left, right, top); + this.drawStrip(asset, top); + this.drawTrack(asset, left, right, top); + const head = this.playhead; + if (head !== null) { + const x = secondsToX(this.view, head); + if (x >= 0 && x <= this.w) this.overlay.rect(x - 0.5, top, 1, TIMELINE_HEIGHT).fill(c.hot); + } + } + + private drawPicture(asset: VideoAsset, top: number): void { + this.player(asset); + const texture = this.preview?.texture; + this.picture.visible = Boolean(texture); + if (!texture) return; + const room = { w: this.w - PREVIEW_PAD * 2, h: top - PREVIEW_PAD * 2 }; + const fitted = containRect(asset.width || texture.width, asset.height || texture.height, room.w, room.h); + this.picture.texture = texture; + this.picture.position.set(PREVIEW_PAD + fitted.x, PREVIEW_PAD + fitted.y); + this.picture.width = Math.max(1, fitted.w); + this.picture.height = Math.max(1, fitted.h); + } + + private drawRuler(left: number, right: number, top: number): void { + const step = rulerStep(this.view.pxPerSecond, LABEL_SPACING); + const placed: { text: string; x: number }[] = []; + const first = Math.ceil(Math.max(0, xToSeconds(this.view, left)) / step); + for (let k = first; ; k++) { + const seconds = k * step; + const x = secondsToX(this.view, seconds); + if (x > right + 0.5) break; + this.overlay.rect(x, top + RULER_HEIGHT - 8, 1, 7).fill({ color: Theme.colors.textMuted, alpha: 0.8 }); + placed.push({ text: rulerLabel(seconds, step), x: x + 3 }); + } + this.syncLabels(placed.map((label) => ({ ...label, y: top + 6 }))); + } + + private litClipId(): string { + const g = this.gesture?.kind === 'span' && this.gesture.moved ? this.gesture : null; + return (g ? g.clipId : this.pickedClip()?.id) ?? ''; + } + + private drawStrip(asset: VideoAsset, top: number): void { + const c = Theme.colors; + const lit = this.litClipId(); + const names = new Map(clipsOf(asset).map((clip) => [clip.id, clip.name])); + const placed: { text: string; x: number; room: number; lit: boolean }[] = []; + const y = top + RULER_HEIGHT; + for (const { id, span } of this.bands(asset)) { + const a = secondsToX(this.view, span.start); + const b = secondsToX(this.view, span.end); + if (b < 0 || a > this.w) continue; + const on = id === lit; + this.overlay + .rect(a, y + 2, Math.max(1, b - a), STRIP_HEIGHT - 4) + .fill({ color: c.accent, alpha: on ? 0.85 : 0.35 }) + .stroke({ width: 1, color: on ? c.accentText : c.accent, alpha: on ? 1 : 0.7, alignment: 1 }); + const name = names.get(id); + if (name) + placed.push({ + text: name, + x: Math.max(a, 0) + 4, + room: Math.min(b, this.w) - Math.max(a, 0) - 8, + lit: on, + }); + } + this.syncStrip(placed.map((label) => ({ ...label, y }))); + } + + /** The video's length lit, what lies outside the picked clip dark, and a handle at each of its edges. */ + private drawTrack(asset: VideoAsset, left: number, right: number, top: number): void { + const c = Theme.colors; + const y = top + RULER_HEIGHT + STRIP_HEIGHT; + const h = TRACK_HEIGHT; + this.overlay.rect(left, y + 4, Math.max(0, right - left), h - 8).fill({ color: c.panel, alpha: 0.8 }); + const clip = this.pickedClip(); + const span = + this.gesture?.kind === 'span' && this.gesture.moved + ? this.gesture.span + : clip && this.clipSpan(clip, asset); + if (!span) return; + const a = secondsToX(this.view, span.start); + const b = secondsToX(this.view, span.end); + this.overlay.rect(a, y + 4, Math.max(1, b - a), h - 8).fill({ color: c.accent, alpha: 0.35 }); + for (const [x, edge] of [ + [a, 'start'], + [b, 'end'], + ] as const) { + const held = this.gesture?.kind === 'span' && this.gesture.edge === edge; + this.overlay.rect(x - 1, y + 2, 2, h - 4).fill(held ? c.hot : c.accent); + const toward = edge === 'start' ? 1 : -1; + this.overlay + .moveTo(x, y + 2) + .lineTo(x + toward * 8, y + 2) + .lineTo(x, y + 12) + .closePath() + .fill(held ? c.hot : c.accent); + } + } + + private syncLabels(placed: readonly { text: string; x: number; y: number }[]): void { + placed.forEach((label, i) => { + let text = this.texts[i]; + if (!text) { + text = crispText({ style: textStyle('small') }); + this.texts.push(text); + this.labels.addChild(text); + } + text.visible = true; + if (text.text !== label.text) text.text = label.text; + text.position.set(Math.round(label.x), Math.round(label.y)); + }); + for (let i = placed.length; i < this.texts.length; i++) this.texts[i]!.visible = false; + } + + private syncStrip( + placed: readonly { text: string; x: number; y: number; room: number; lit: boolean }[], + ): void { + placed.forEach((label, i) => { + let text = this.stripTexts[i]; + if (!text) { + text = crispText({ style: textStyle('small') }); + this.stripTexts.push(text); + this.stripLabels.addChild(text); + } + const style = label.lit ? LIT_LABEL : textStyle('small'); + if (text.style !== style) text.style = style; + if (text.text !== label.text) text.text = label.text; + for (let cut = label.text.length; text.width > label.room && cut > 1; cut--) + text.text = `${label.text.slice(0, cut - 1)}…`; + text.visible = text.width <= label.room; + text.position.set(Math.round(label.x), Math.round(label.y + (STRIP_HEIGHT - text.height) / 2)); + }); + for (let i = placed.length; i < this.stripTexts.length; i++) this.stripTexts[i]!.visible = false; + } +} diff --git a/src/editor/video/VideoEditor.ts b/src/editor/video/VideoEditor.ts new file mode 100644 index 00000000..a66e7348 --- /dev/null +++ b/src/editor/video/VideoEditor.ts @@ -0,0 +1,84 @@ +import type { Asset, VideoAsset } from '../../core/model/Asset'; +import { Button } from '../../ui'; +import { AssetTabEditor } from '../AssetTabEditor'; +import { TOOLBAR_ICON_SVG } from '../panels/toolbarIcons'; +import type { EditorState } from '../state'; +import { VideoCanvas } from './VideoCanvas'; +import { formatDuration } from './videoInfo'; + +const INFO_WIDTH = 220; + +/** + * The Video tab (#402): the video with its clips, the stretches of it a sprite can wear instead of + * the whole video. The header plays the picked clip, else the whole video (Space does too). + */ +export class VideoEditor extends AssetTabEditor { + /** Public for the check scripts. */ + readonly playButton: Button; + + constructor(state: EditorState) { + super(state, { infoWidth: INFO_WIDTH }); + this.playButton = new Button('Play', { + icon: TOOLBAR_ICON_SVG.play, + minWidth: 72, + tooltip: 'Play the picked clip, else the whole video (Space)', + onClick: () => this.togglePlay(), + }); + const canvas = new VideoCanvas(state); + canvas.onTogglePlay = () => this.togglePlay(); + canvas.onStopped = () => this.syncPlayButton(); + this.mount(canvas, [this.playButton]); + state.on('load', () => this.canvas.stop()); + state.on('selection', () => this.refresh()); + } + + override selectedId(): string { + return this.state.selectedVideoId ?? ''; + } + + override isMine(asset: Asset): asset is VideoAsset { + return asset.type === 'video'; + } + + get isPlaying(): boolean { + return this.canvas.isPlaying; + } + + togglePlay(): void { + if (this.canvas.isPlaying) this.canvas.stop(); + else this.canvas.play(); + this.syncPlayButton(); + } + + /** Only called while the tab is visible. */ + tick(): void { + this.canvas.tick(); + } + + protected override closed(assetId: string): void { + if (this.canvas.currentAssetId === assetId) this.canvas.stop(); + } + + protected override infoText(asset: VideoAsset): string { + if (!asset.file) return 'Not uploaded'; + const span = this.canvas.shownSpan(); + const whole = formatDuration(asset.duration); + const length = + span && span.end - span.start < asset.duration + ? `${formatDuration(span.end - span.start)} of ${whole}` + : whole; + return `${asset.width} × ${asset.height} · ${length} · ${Number(asset.fps.toFixed(2))} fps`; + } + + protected override syncExtras(asset: VideoAsset | null): void { + this.fitButton.setEnabled(Boolean(asset?.file)); + this.syncPlayButton(); + } + + private syncPlayButton(): void { + const playing = this.canvas.isPlaying; + this.playButton.setIcon(playing ? TOOLBAR_ICON_SVG.stop : TOOLBAR_ICON_SVG.play); + this.playButton.setText(playing ? 'Stop' : 'Play'); + this.playButton.setEnabled(Boolean(this.asset()?.file)); + } +} diff --git a/src/editor/video/VideoPicker.ts b/src/editor/video/VideoPicker.ts new file mode 100644 index 00000000..0fd6f9ab --- /dev/null +++ b/src/editor/video/VideoPicker.ts @@ -0,0 +1,185 @@ +import { Container, Graphics, Sprite, type Text } from 'pixi.js'; +import type { StoredVideo } from '../../core/store/VideoStore'; +import { crispText, formatBytes, textStyle, truncate } from '../../ui'; +import { posterLibrary } from '../../viewport/VideoLibrary'; +import { pickFiles } from '../assetImport'; +import { + currentPicker, + FilePicker, + openPicker, + type PickerItem, + type PickerRow, + RowList, + withAssetNames, +} from '../FilePicker'; +import { containRect } from '../texture/imageGrid'; +import { importStoredVideo, uploadVideos, VIDEO_ACCEPT, type VideoImportDeps } from '../videoImport'; +import { isPending, videoStatusText } from './videoInfo'; + +const ROW_HEIGHT = 52; +const POSTER_W = 80; +const POSTER_H = 45; +const ROW_PAD = 8; +/** How often the list is asked again while a video is on its way. */ +const POLL_MS = 1000; + +type VideoItem = PickerItem; +interface VideoRow extends PickerRow { + poster: Sprite; + frame: Graphics; + detail: Text; +} + +/** One row per video: its poster, its name, and what it is or how far its conversion is. */ +class VideoList extends RowList { + protected readonly rowHeight = ROW_HEIGHT; + + constructor() { + super(); + this.watch(posterLibrary, () => this.redraw()); + } + + protected buildRow(): VideoRow { + const box = new Container(); + const background = new Graphics(); + const frame = new Graphics(); + const poster = new Sprite(); + const caption = crispText({ style: textStyle('body') }); + const detail = crispText({ style: textStyle('small') }); + box.addChild(background, frame, poster, caption, detail); + return { box, background, frame, poster, caption, detail }; + } + + protected redrawRow(row: VideoRow, item: VideoItem, selected: boolean): void { + this.drawBox(row, this.w, ROW_HEIGHT, selected); + const top = (ROW_HEIGHT - POSTER_H) / 2; + row.frame.clear().rect(ROW_PAD, top, POSTER_W, POSTER_H).fill(0x000000); + const texture = item.status === 'ready' && item.poster ? posterLibrary.load(item.poster) : null; + row.poster.visible = texture !== null; + if (texture) { + const fitted = containRect(texture.width, texture.height, POSTER_W, POSTER_H); + row.poster.texture = texture; + row.poster.position.set(ROW_PAD + fitted.x, top + fitted.y); + row.poster.width = fitted.w; + row.poster.height = fitted.h; + } + + const left = ROW_PAD * 2 + POSTER_W; + const width = this.w - left - ROW_PAD; + row.caption.style = textStyle(item.assetName ? 'accent' : 'body'); + row.caption.text = truncate(item.assetName ?? (item.name || 'Video'), row.caption.style, width); + row.caption.position.set(left, 8); + row.detail.style = textStyle(item.status === 'failed' ? 'danger' : 'small'); + row.detail.text = truncate(videoStatusText(item), row.detail.style, width); + row.detail.position.set(left, ROW_HEIGHT - 20); + } +} + +/** + * "+ → Video" (#402). An upload is converted on the server before it can be used, so the list + * shows the ones on their way with their progress and asks again until none is left. What was + * uploaded from here is made an asset the moment it is ready. + */ +class VideoPicker extends FilePicker { + declare protected readonly list: VideoList; + private poll: ReturnType | null = null; + private lastItems: VideoItem[] = []; + + constructor( + private readonly deps: VideoImportDeps, + /** Uploads to adopt once they are ready; the dialog closes when the last one is. */ + private readonly waiting: Set, + ) { + super( + new VideoList(), + { + title: 'Choose a video', + loading: 'Loading videos…', + idle: 'Videos are converted to MP4 on the server, up to 1080p.', + empty: 'No videos on the server yet – upload one.', + use: 'Use video', + noun: 'video', + }, + () => uploadFromDialog(deps), + (file) => deps.videos.delete(file), + ); + } + + /** Global centre of a row (test hook). */ + rowGlobal(file: string): { x: number; y: number } | null { + return this.list.rowGlobal(file); + } + + /** The uploads it waits for (test hook). */ + get waitingFor(): string[] { + return [...this.waiting]; + } + + override close(): void { + if (this.poll !== null) clearTimeout(this.poll); + this.poll = null; + super.close(); + } + + override async load(): Promise { + await super.load(); + if (this.list.destroyed) return; + // What was just uploaded is what the dialog is open for: shown with its progress. + const first = [...this.waiting][0]; + if (!this.list.selected && first) this.list.select(first); + const items = this.list.fileList.map((file) => this.itemOf(file)).filter((item) => item !== null); + for (const item of items) { + if (!this.waiting.has(item.file) || isPending(item)) continue; + this.waiting.delete(item.file); + if (item.status === 'ready') importStoredVideo(item, this.deps); + else this.list.select(item.file); + if (this.waiting.size === 0 && item.status === 'ready') { + this.close(); + return; + } + } + if (items.some(isPending)) this.poll = setTimeout(() => void this.load(), POLL_MS); + } + + protected async fetchItems(): Promise { + const items = withAssetNames(await this.deps.videos.list(), this.deps.state.project, 'video'); + this.lastItems = items; + return items; + } + + protected contentHeight(count: number): { unit: number; total: number } { + return { unit: ROW_HEIGHT, total: this.list.heightOfRows(count) }; + } + + protected adopt(item: VideoItem): void { + importStoredVideo(item, this.deps); + } + + protected override canUse(item: VideoItem): boolean { + return item.status === 'ready'; + } + + protected override describe(item: VideoItem): string { + const size = item.status === 'ready' ? ` – ${formatBytes(item.bytes)}` : ''; + return `${item.name || 'Video'}${size} – ${videoStatusText(item)}`; + } + + private itemOf(file: string): VideoItem | null { + return this.lastItems.find((item) => item.file === file) ?? null; + } +} + +/** The dialog's Upload… button: the file dialog, the uploads, then the picker again to watch them. */ +function uploadFromDialog(deps: VideoImportDeps): void { + void pickFiles(VIDEO_ACCEPT) + .then((files) => uploadVideos(files, deps, (pending) => openVideoPicker(deps, pending))) + .catch((error: unknown) => deps.state.setNotice(String(error), 'error')); +} + +export function openVideoPicker(deps: VideoImportDeps, waiting: readonly string[] = []): VideoPicker { + return openPicker(new VideoPicker(deps, new Set(waiting))); +} + +export function currentVideoPicker(): VideoPicker | null { + return currentPicker(VideoPicker); +} diff --git a/src/editor/video/videoInfo.ts b/src/editor/video/videoInfo.ts new file mode 100644 index 00000000..944bc7a8 --- /dev/null +++ b/src/editor/video/videoInfo.ts @@ -0,0 +1,36 @@ +import type { UploadedVideo } from '../../core/store/VideoStore'; + +/** "12.5 s", "1:05", "1:02:03": as long as it needs to be. */ +export function formatDuration(seconds: number): string { + const s = Math.max(0, seconds); + if (s < 60) return `${Number(s.toFixed(1))} s`; + const whole = Math.round(s); + const h = Math.floor(whole / 3600); + const m = Math.floor((whole % 3600) / 60); + const rest = String(whole % 60).padStart(2, '0'); + return h > 0 ? `${h}:${String(m).padStart(2, '0')}:${rest}` : `${m}:${rest}`; +} + +/** What a video is, or where it is on its way there (#402) – the picker's second line. */ +export function videoStatusText(video: UploadedVideo): string { + switch (video.status) { + case 'queued': + return 'Waiting to be converted…'; + case 'converting': + return `Converting… ${Math.round((video.progress ?? 0) * 100)} %`; + case 'failed': + return `Could not be converted: ${video.error || 'no reason given'}`; + default: + return [ + `${video.width ?? 0} × ${video.height ?? 0}`, + formatDuration(video.duration ?? 0), + `${Number((video.fps ?? 0).toFixed(2))} fps`, + video.audio ? 'with sound' : 'silent', + ].join(' · '); + } +} + +/** Whether a listed video is still on its way, so the list is asked again. */ +export function isPending(video: UploadedVideo): boolean { + return video.status === 'queued' || video.status === 'converting'; +} diff --git a/src/editor/videoImport.ts b/src/editor/videoImport.ts new file mode 100644 index 00000000..15dea713 --- /dev/null +++ b/src/editor/videoImport.ts @@ -0,0 +1,77 @@ +import { CreateVideoCommand } from '../core/commands/assetCommands'; +import type { UploadedVideo, VideoStore } from '../core/store/VideoStore'; +import { type ImportKind, type ImportOutcome, importStored } from './assetImport'; +import type { EditorState } from './state'; + +/** What the file dialog offers and a drop accepts; ffmpeg on the server has the last word. */ +export const VIDEO_ACCEPT = '.mp4,.mov,.m4v,.webm,.mkv,.avi'; + +const VIDEO_EXTENSION = /\.(mp4|mov|m4v|webm|mkv|avi)$/i; + +export function isVideoFileName(name: string): boolean { + return VIDEO_EXTENSION.test(name); +} + +export interface VideoImportDeps { + state: EditorState; + videos: VideoStore; +} + +/** + * A video becomes an asset only once the worker has made it (#402): the asset copies its size, + * length and frame rate, which are not known before. + */ +const VIDEO_KIND: ImportKind = { + type: 'video', + noun: 'video', + fallbackName: 'Video', + store: (deps) => deps.videos, + prepare: (file) => + isVideoFileName(file.name) + ? Promise.resolve({ blob: file, type: 'application/octet-stream' }) + : Promise.reject(new Error('Not a video file – use MP4, MOV, WebM, MKV or AVI.')), + create: (name, video) => + new CreateVideoCommand(name, video.file, { + width: video.width ?? 0, + height: video.height ?? 0, + duration: video.duration ?? 0, + fps: video.fps ?? 0, + audio: video.audio ?? false, + }), +}; + +export function importStoredVideo(video: UploadedVideo, deps: VideoImportDeps): ImportOutcome { + return importStored(VIDEO_KIND, video, deps); +} + +/** + * Sends the files up one after the other and says how it went; what is done already is made an + * asset at once. The rest is converted on the server – `waiting` gets their names, for the picker + * to watch and adopt when they are ready. + */ +export async function uploadVideos( + files: readonly File[], + deps: VideoImportDeps, + waiting: (files: string[]) => void, +): Promise { + const pending: string[] = []; + const failed: string[] = []; + for (const file of files) { + try { + const { blob, type } = await VIDEO_KIND.prepare(file); + const uploaded = await deps.videos.upload(blob, type, file.name); + if (uploaded.status === 'ready') importStoredVideo(uploaded, deps); + else pending.push(uploaded.file); + } catch (error) { + failed.push(`"${file.name}": ${error instanceof Error ? error.message : String(error)}`); + } + } + if (pending.length > 0) waiting(pending); + if (failed.length > 0) deps.state.setNotice(`Could not upload ${failed.join(', ')}`, 'error'); + else if (pending.length > 0) + deps.state.setNotice( + pending.length === 1 + ? 'Converting the video on the server…' + : `Converting ${pending.length} videos on the server…`, + ); +} diff --git a/src/player/Player.ts b/src/player/Player.ts index 3cf7c195..7f13effd 100644 --- a/src/player/Player.ts +++ b/src/player/Player.ts @@ -7,6 +7,7 @@ import { behaviorError } from '../runtime/systems/BehaviorSystem'; import { Input } from '../runtime/systems/Input'; import { requestRender } from '../ui'; import { soundLibrary } from '../viewport/SoundLibrary'; +import { videoLibrary } from '../viewport/VideoLibrary'; import type { GameScreen } from './GameScreen'; /** @@ -57,6 +58,7 @@ export class Player { clock: gameClock, saves: this.saves, }); + videoLibrary.startRun(); this.input.attach(); this.screen.setWorld(this.run.world); if (this.run.world.stopRequested) this.stop(); @@ -68,6 +70,7 @@ export class Player { this.run = null; this.input.detach(); soundLibrary.stopAll(); + videoLibrary.endRun(); gameClock.reset(); const error = behaviorError(run.world); if (error) console.error('[game]', error); diff --git a/src/viewport/SoundLibrary.ts b/src/viewport/SoundLibrary.ts index bfcb2e07..6553d91f 100644 --- a/src/viewport/SoundLibrary.ts +++ b/src/viewport/SoundLibrary.ts @@ -1658,6 +1658,20 @@ export class SoundLibrary implements AudioSink { return this.live()!; } + /** + * The sound of a video (#402) into the master, as a voice goes: from now on `element` is heard + * through the mix and only through it. The gain is its volume. Null while a bake has the context. + */ + mediaInput(element: HTMLMediaElement): GainNode | null { + this.audio(); + const context = this.live(); + if (!context) return null; + if (context.state === 'suspended' && !this.held) void context.resume(); + const gain = context.createGain(); + context.createMediaElementSource(element).connect(gain).connect(this.busInput('')); + return gain; + } + /** The context it plays to the speakers on; null before there is one, and for a bake (#340). */ private live(): AudioContext | null { return this.render ? null : (this.context as AudioContext | null); diff --git a/src/viewport/TextureLibrary.ts b/src/viewport/TextureLibrary.ts index 9ea7ac02..d66865fc 100644 --- a/src/viewport/TextureLibrary.ts +++ b/src/viewport/TextureLibrary.ts @@ -1,9 +1,11 @@ import { Assets, CanvasSource, Texture } from 'pixi.js'; import { isGeneratedFile } from '../core/model/generated'; +import { isVideoKey } from '../core/model/video'; import type { FrameRect, TextureRef } from '../core/model/texture'; import { type EdgeMode, SHEET_EDGE } from '../core/shader/nodes'; import { FileLibrary } from './FileLibrary'; import { GeneratedPictures, type PictureFactory } from './generatedPictures'; +import { videoLibrary } from './VideoLibrary'; /** * Pixi turns `clamp-to-edge` into `repeat` on every texture a Graphics fill draws with @@ -45,6 +47,7 @@ export class TextureLibrary extends FileLibrary { constructor() { super('/game-maker/data/textures', 'textures'); + videoLibrary.onChange(() => this.notifyChange()); } /** Installs who draws generated pictures; without one, a generated key stays empty. */ @@ -72,6 +75,8 @@ export class TextureLibrary extends FileLibrary { get(ref: TextureRef, edge: EdgeMode = SHEET_EDGE): Texture | null { const sheet = this.load(ref.file); if (!sheet) return null; + // A video (#402) is a picture that changes every frame; a cut of it would stand still. + if (isVideoKey(ref.file)) return sheet; const rect = ref.rect ?? (edge === SHEET_EDGE ? null : wholeOf(sheet)); const texture = rect ? this.crop(ref.file, sheet, rect, edge) : sheet; if (!texture) return null; @@ -92,6 +97,7 @@ export class TextureLibrary extends FileLibrary { */ load(file: string): Texture | null { if (isGeneratedFile(file)) return this.generated.get(file); + if (isVideoKey(file)) return videoLibrary.get(file); const texture = this.loaded.get(file); if (!texture) { this.startLoading(file); @@ -103,6 +109,7 @@ export class TextureLibrary extends FileLibrary { /** The picture, or one of its cut frames, if in memory – without a load or a scale-mode change. */ peek(file: string, rect: FrameRect | null = null): Texture | null { if (rect) return this.crops.get(cropKey(file, rect)) ?? null; + if (isVideoKey(file)) return videoLibrary.get(file); return this.loaded.get(file) ?? this.generated.peek(file); } @@ -140,7 +147,7 @@ export class TextureLibrary extends FileLibrary { * loads, and for a generated picture, which lives on the GPU only. */ pixels(file: string): ImageData | null { - const sheet = isGeneratedFile(file) ? null : this.load(file); + const sheet = isGeneratedFile(file) || isVideoKey(file) ? null : this.load(file); const picture = sheet?.source.resource as CanvasImageSource | null | undefined; if (!sheet || !picture) return null; const { pixelWidth: w, pixelHeight: h } = sheet.source; diff --git a/src/viewport/VideoLibrary.ts b/src/viewport/VideoLibrary.ts new file mode 100644 index 00000000..1bd6305f --- /dev/null +++ b/src/viewport/VideoLibrary.ts @@ -0,0 +1,259 @@ +/** + * The players behind every video a sprite wears (#402): one `