diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 46b889d5..e9ba9634 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -43,6 +43,7 @@ jobs: php tests/php/snapshots.test.php php tests/php/releases.test.php php tests/php/files.test.php + php tests/php/videos.test.php suite: runs-on: game-maker-wsl diff --git a/.gitignore b/.gitignore index 0f971aa8..b3b65338 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ /data/textures/ /data/fonts/ /data/sounds/ +/data/videos/ /data/pixels/ /data/plugins/ /dist/ diff --git a/README.md b/README.md index 7d64e470..736484ff 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh The browser suite and the three example games need a machine with a GPU: ```bash -node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 1976 checks (GM_STEPS=20 for one step) +node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 1994 checks (GM_STEPS=20 for one step) # every tool signs in first – see the Accounts page node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs diff --git a/api/index.php b/api/index.php index d211942f..bb102a77 100644 --- a/api/index.php +++ b/api/index.php @@ -63,6 +63,13 @@ declare(strict_types=1); * POST ?r=sounds&name=x.wav raw sound bytes → {file,name,bytes} * DELETE ?r=sounds/{file} → 204 | 403 | 404 | 409 * + * GET ?r=videos → [{file,name,status,…,owner,deletable}] + * POST ?r=videos&name=x.mov raw video bytes → {file,name,status,…} + * DELETE ?r=videos/{file} → 204 | 403 | 404 | 409 + * (an upload is converted by the worker in `game-maker-video` (#402): + * `status` is queued, converting with `progress`, failed with `error`, + * or ready with width, height, duration, fps, audio and `poster`) + * * Uploads land in the uploader's own folder, and `file` carries it: `{folder}/{sha256}.{ext}` * (#344). A `file` without a folder belongs to nobody. A list shows the files of the one asking and * nobody's, an administrator's every file; `owner` is the account a file belongs to (null for @@ -78,8 +85,8 @@ declare(strict_types=1); * empty when none is known. It is a caption, never a path: the stored file is always `{hash}.{ext}` * in a folder. * - * Stored images, fonts, sounds and cels are served straight from `data/textures/`, `data/fonts/`, - * `data/sounds/` and `data/pixels/` as static files; the API only writes them. + * Stored images, fonts, sounds, videos and cels are served straight from `data/textures/`, `data/fonts/`, + * `data/sounds/`, `data/videos/` and `data/pixels/` as static files; the API only writes them. */ require __DIR__ . '/lib/AccountRules.php'; @@ -99,8 +106,13 @@ require __DIR__ . '/lib/SoundStore.php'; require __DIR__ . '/lib/Store.php'; require __DIR__ . '/lib/TextureStore.php'; require __DIR__ . '/lib/UserStore.php'; +require __DIR__ . '/lib/VideoQueue.php'; +require __DIR__ . '/lib/VideoStore.php'; -/** Has to clear the largest single upload below, which is a sound; the web server has its own say. */ +/** + * Has to clear the largest upload held in memory, which is a sound; a video is streamed to disk and + * has its own ceiling. The web server and the proxy in front of it have their own say. + */ const MAX_BODY_BYTES = 256 * 1024 * 1024; /** Upload ceiling for a single image. The editor downscales larger pictures before sending. */ @@ -112,6 +124,12 @@ const MAX_FONT_BYTES = 4 * 1024 * 1024; /** The same for a sound, far higher: a recorded take is plain WAV, about 46 minutes in mono. */ const MAX_SOUND_BYTES = 256 * 1024 * 1024; +/** + * A video goes up as it was recorded and is made smaller by the worker, not on the way. It is + * streamed to disk, so memory is no limit; nginx and the Brandmauer let 2 GB through (#402). + */ +const MAX_VIDEO_BYTES = 2048 * 1024 * 1024; + set_error_handler(static function (int $severity, string $message, string $file, int $line): bool { // PHP calls the handler for a call silenced with `@` as well, only with less reported: that // warning was expected and is answered where it happens (an unreadable image is a 422). @@ -514,12 +532,21 @@ function routeFiles( Response::json($store->list($users->folderOf($user), $isAdmin, $users->folderOwners())); } if ($method === 'POST') { - $bytes = Response::rawBody(); - if (strlen($bytes) > $maxBytes) { - Response::error("{$noun} must be at most " . ($maxBytes / 1024 / 1024) . ' MB', 413); - } + $tooLarge = "{$noun} must be at most " . ($maxBytes / 1024 / 1024) . ' MB'; + $name = (string) ($_GET['name'] ?? ''); try { - Response::json($store->store($bytes, (string) ($_GET['name'] ?? ''), $users->folderOf($user, true)), 201); + // A video is too big for memory and goes from the request straight to disk. + if ($store instanceof VideoStore) { + $in = fopen('php://input', 'rb'); + Response::json($store->storeStream($in, $maxBytes, $name, $users->folderOf($user, true)), 201); + } + $bytes = Response::rawBody(); + if (strlen($bytes) > $maxBytes) { + Response::error($tooLarge, 413); + } + Response::json($store->store($bytes, $name, $users->folderOf($user, true)), 201); + } catch (VideoTooLargeException) { + Response::error($tooLarge, 413); } catch (RuntimeException $e) { if (!$e instanceof $invalid) { throw $e; @@ -600,19 +627,20 @@ function routePlugins(array $parts, string $method): never } try { - if ((int) ($_SERVER['CONTENT_LENGTH'] ?? 0) > MAX_BODY_BYTES) { - Response::error('Request body too large', 413); - } - $method = $_SERVER['REQUEST_METHOD'] ?? 'GET'; $route = trim((string) ($_GET['r'] ?? ''), '/'); $parts = $route === '' ? [] : explode('/', $route); + $maxBody = ($parts[0] ?? '') === 'videos' ? MAX_VIDEO_BYTES : MAX_BODY_BYTES; + if ((int) ($_SERVER['CONTENT_LENGTH'] ?? 0) > $maxBody) { + Response::error('Request body too large', 413); + } + // A project reaches deepest, `projects/{id}/snapshots/{snapshotId}/restore`; a stored file is // `textures/{folder}/{file}`; everything else stops at two parts. $depth = match ($parts[0] ?? '') { 'projects' => 5, - 'textures', 'fonts', 'sounds' => 3, + 'textures', 'fonts', 'sounds', 'videos' => 3, default => 2, }; if (count($parts) < 1 || count($parts) > $depth) { @@ -671,6 +699,17 @@ try { 'Sound', InvalidSoundException::class, ), + 'videos' => routeFiles( + $parts, + $method, + $user, + $isAdmin, + $users, + new VideoStore("{$data}/videos"), + MAX_VIDEO_BYTES, + 'Video', + InvalidVideoException::class, + ), 'pixels' => routePixels($parts, $method), 'plugins' => routePlugins($parts, $method), default => Response::error('Not found', 404), diff --git a/api/lib/VideoFormat.php b/api/lib/VideoFormat.php new file mode 100644 index 00000000..c1e57883 --- /dev/null +++ b/api/lib/VideoFormat.php @@ -0,0 +1,216 @@ + $probe ffprobe's `-show_streams -show_format` as JSON + * @return array{width:int,height:int,fps:string,audio:bool,duration:float} + * @throws InvalidArgumentException when there is no picture in it + */ + public static function plan(array $probe): array + { + $streams = is_array($probe['streams'] ?? null) ? $probe['streams'] : []; + $video = null; + $audio = false; + foreach ($streams as $stream) { + $type = $stream['codec_type'] ?? ''; + // A cover picture in a music file is a video stream as well, one that is a single frame. + $still = (int) (($stream['disposition'] ?? [])['attached_pic'] ?? 0) === 1; + if ($type === 'video' && $video === null && !$still) { + $video = $stream; + } + $audio = $audio || $type === 'audio'; + } + if ($video === null || (int) ($video['width'] ?? 0) <= 0 || (int) ($video['height'] ?? 0) <= 0) { + throw new InvalidArgumentException('There is no picture in this file'); + } + + [$width, $height] = [(int) $video['width'], (int) $video['height']]; + if (abs(self::rotation($video)) % 180 === 90) { + [$width, $height] = [$height, $width]; + } + [$width, $height] = self::fit($width, $height); + + return [ + 'width' => $width, + 'height' => $height, + 'fps' => self::fps($video), + 'audio' => $audio, + 'duration' => self::duration($probe, $video), + ]; + } + + /** + * The size inside the box, keeping the shape, both sides even (H.264 in `yuv420p` needs that). + * + * @return array{0:int,1:int} + */ + public static function fit(int $width, int $height): array + { + [$boxW, $boxH] = $width >= $height + ? [self::MAX_LONG_SIDE, self::MAX_SHORT_SIDE] + : [self::MAX_SHORT_SIDE, self::MAX_LONG_SIDE]; + $scale = min(1.0, $boxW / $width, $boxH / $height); + $even = static fn(float $side): int => max(2, 2 * (int) floor(round($side, 3) / 2)); + return [$even($width * $scale), $even($height * $scale)]; + } + + /** + * The frame rate the file is written at: its own average, unless that is above 60 or unknown. + * Kept as ffmpeg's fraction, so 29.97 stays 30000/1001 and frames do not drift against the time. + * + * @param array $video + */ + public static function fps(array $video): string + { + foreach (['avg_frame_rate', 'r_frame_rate'] as $key) { + $rate = self::rate((string) ($video[$key] ?? '')); + if ($rate !== null) { + return $rate[1] > self::MAX_FPS ? (string) self::MAX_FPS : $rate[0]; + } + } + return self::FALLBACK_FPS; + } + + /** A fraction such as `30000/1001` as a number of frames per second. */ + public static function fpsValue(string $fps): float + { + $rate = self::rate($fps); + return $rate === null ? 0.0 : $rate[1]; + } + + /** + * @param array{width:int,height:int,fps:string,audio:bool} $plan + * @return list + */ + public static function convertArgs(string $source, string $target, array $plan): array + { + $args = [ + 'ffmpeg', '-nostdin', '-hide_banner', '-loglevel', 'error', '-y', + '-i', $source, + '-map', '0:v:0', + ]; + if ($plan['audio']) { + $args = [...$args, '-map', '0:a:0']; + } + $args = [ + ...$args, + '-vf', "scale={$plan['width']}:{$plan['height']}:flags=lanczos,setsar=1", + '-r', $plan['fps'], '-fps_mode', 'cfr', + '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '21', + '-pix_fmt', 'yuv420p', + '-g', (string) self::KEYFRAME_EVERY, '-keyint_min', (string) self::KEYFRAME_EVERY, '-sc_threshold', '0', + ]; + $args = $plan['audio'] + ? [...$args, '-c:a', 'aac', '-b:a', '160k', '-ac', '2', '-ar', '48000'] + : [...$args, '-an']; + return [...$args, '-sn', '-dn', '-map_metadata', '-1', '-movflags', '+faststart', '-progress', 'pipe:1', '-nostats', '-f', 'mp4', $target]; + } + + /** + * A still from a little way in (not the very first frame, which is often black), as wide as the + * picker draws it. + * + * @return list + */ + public static function posterArgs(string $video, string $target, float $duration): array + { + $at = sprintf('%.3f', max(0.0, min(1.0, $duration / 2))); + return [ + 'ffmpeg', '-nostdin', '-hide_banner', '-loglevel', 'error', '-y', + '-ss', $at, '-i', $video, '-frames:v', '1', + '-vf', 'scale=320:-2', '-q:v', '4', '-f', 'image2', $target, + ]; + } + + /** @return list */ + public static function probeArgs(string $path): array + { + return ['ffprobe', '-v', 'error', '-print_format', 'json', '-show_streams', '-show_format', $path]; + } + + /** + * What the editor is told about a converted file, read back from the file itself rather than + * the plan: a crop or a rounding inside ffmpeg is what the editor has to draw. + * + * @param array $probe + * @return array{width:int,height:int,duration:float,fps:float,audio:bool} + */ + public static function details(array $probe): array + { + $plan = self::plan($probe); + return [ + 'width' => $plan['width'], + 'height' => $plan['height'], + 'duration' => round($plan['duration'], 3), + 'fps' => round(self::fpsValue($plan['fps']), 3), + 'audio' => $plan['audio'], + ]; + } + + /** The share of the work done, from one `out_time_us=` line of ffmpeg's progress. */ + public static function progress(string $line, float $duration): ?float + { + if ($duration <= 0 || !preg_match('/^out_time_us=(\d+)$/', trim($line), $m)) { + return null; + } + return max(0.0, min(1.0, ((int) $m[1]) / 1e6 / $duration)); + } + + /** @param array $video */ + private static function rotation(array $video): int + { + foreach (is_array($video['side_data_list'] ?? null) ? $video['side_data_list'] : [] as $side) { + if (isset($side['rotation']) && is_numeric($side['rotation'])) { + return (int) $side['rotation']; + } + } + $tag = ($video['tags'] ?? [])['rotate'] ?? null; + return is_numeric($tag) ? (int) $tag : 0; + } + + /** @param array $probe @param array $video */ + private static function duration(array $probe, array $video): float + { + foreach ([$video['duration'] ?? null, ($probe['format'] ?? [])['duration'] ?? null] as $value) { + if (is_numeric($value) && (float) $value > 0) { + return (float) $value; + } + } + return 0.0; + } + + /** @return array{0:string,1:float}|null the fraction as written and its value, null for none */ + private static function rate(string $text): ?array + { + if (!preg_match('#^(\d+)(?:/(\d+))?$#', $text, $m)) { + return null; + } + $num = (int) $m[1]; + $den = isset($m[2]) ? (int) $m[2] : 1; + if ($num <= 0 || $den <= 0) { + return null; + } + return [$den === 1 ? (string) $num : "{$num}/{$den}", $num / $den]; + } +} diff --git a/api/lib/VideoQueue.php b/api/lib/VideoQueue.php new file mode 100644 index 00000000..4ba76658 --- /dev/null +++ b/api/lib/VideoQueue.php @@ -0,0 +1,173 @@ +dir) && !mkdir($this->dir, 0775, true) && !is_dir($this->dir)) { + throw new RuntimeException("Cannot create directory {$this->dir}"); + } + } + + /** + * Copies an upload into a temp file beside the jobs, never more than `$maxBytes` of it: a video + * is too big to be held in memory. The caller hands the file to add() or deletes it. + * + * @param resource $in + * @throws VideoTooLargeException when the stream has more than `$maxBytes` + */ + public function receive($in, int $maxBytes): string + { + $tmp = "{$this->dir}/upload-" . bin2hex(random_bytes(8)) . '.tmp'; + $out = fopen($tmp, 'xb'); + if ($out === false) { + throw new RuntimeException('Cannot write uploaded video'); + } + $copied = stream_copy_to_stream($in, $out, $maxBytes + 1); + fclose($out); + if ($copied === false || $copied > $maxBytes) { + unlink($tmp); + throw $copied === false ? new RuntimeException('Cannot write uploaded video') : new VideoTooLargeException(); + } + return $tmp; + } + + /** Queues the upload received into `$upload`, which is moved and gone afterwards. */ + public function add(string $folder, string $hash, string $upload, string $name): void + { + if (!chmod($upload, AtomicFile::HIDDEN) || !rename($upload, $this->source($folder, $hash))) { + throw new RuntimeException('Cannot write uploaded video'); + } + $this->write($folder, $hash, [ + 'folder' => $folder, + 'hash' => $hash, + 'name' => $name, + 'state' => self::QUEUED, + 'progress' => 0, + 'error' => '', + 'createdAt' => gmdate('c'), + ]); + } + + /** @return array|null */ + public function job(string $folder, string $hash): ?array + { + $path = $this->jobPath($folder, $hash); + if (!is_file($path)) { + return null; + } + $raw = @file_get_contents($path); + $job = $raw === false ? null : json_decode($raw, true); + return is_array($job) ? $job : null; + } + + /** @return list> every job, oldest first */ + public function jobs(): array + { + $jobs = []; + foreach (glob($this->dir . '/*.json') ?: [] as $path) { + if (!preg_match('/^(' . UserStore::FOLDER_PATTERN . ')-([a-f0-9]{64})\.json$/', basename($path), $m)) { + continue; + } + $job = $this->job($m[1], $m[2]); + if ($job !== null) { + $jobs[] = $job; + } + } + usort($jobs, static fn(array $a, array $b): int => strcmp((string) $a['createdAt'], (string) $b['createdAt'])); + return $jobs; + } + + /** The oldest job nobody has started on, or null when there is none. */ + public function next(): ?array + { + foreach ($this->jobs() as $job) { + if ($job['state'] === self::QUEUED && is_file($this->source($job['folder'], $job['hash']))) { + return $job; + } + } + return null; + } + + /** + * Changes a job, unless it was deleted in the meantime: then false, and the one converting it + * knows to stop. + * + * @param array $changes + */ + public function update(string $folder, string $hash, array $changes): bool + { + $job = $this->job($folder, $hash); + if ($job === null) { + return false; + } + $this->write($folder, $hash, $changes + $job); + return true; + } + + /** A job that was being converted when the worker stopped is started again. */ + public function requeueInterrupted(): int + { + $count = 0; + foreach ($this->jobs() as $job) { + if ($job['state'] === self::CONVERTING) { + $this->update($job['folder'], $job['hash'], ['state' => self::QUEUED, 'progress' => 0]); + $count++; + } + } + return $count; + } + + public function remove(string $folder, string $hash): bool + { + $removed = false; + foreach ([$this->source($folder, $hash), $this->jobPath($folder, $hash)] as $path) { + if (is_file($path)) { + $removed = unlink($path) || $removed; + } + } + return $removed; + } + + /** Drops the upload once it is converted or cannot be; the job itself says what happened. */ + public function dropSource(string $folder, string $hash): void + { + $path = $this->source($folder, $hash); + if (is_file($path)) { + unlink($path); + } + } + + public function source(string $folder, string $hash): string + { + return "{$this->dir}/{$folder}-{$hash}.src"; + } + + private function jobPath(string $folder, string $hash): string + { + return "{$this->dir}/{$folder}-{$hash}.json"; + } + + /** @param array $job */ + private function write(string $folder, string $hash, array $job): void + { + $json = json_encode($job, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); + AtomicFile::write($this->jobPath($folder, $hash), (string) $json, AtomicFile::HIDDEN, 'video job'); + } +} diff --git a/api/lib/VideoStore.php b/api/lib/VideoStore.php new file mode 100644 index 00000000..9039228c --- /dev/null +++ b/api/lib/VideoStore.php @@ -0,0 +1,234 @@ +queue = new VideoQueue("{$root}/queue"); + } + + protected static function filePattern(): string + { + return '[a-f0-9]{64}\.mp4'; + } + + public function queue(): VideoQueue + { + return $this->queue; + } + + /** + * Takes an upload: a file already converted answers as it is, anything else waits in the + * queue, a failed one is tried again. + * + * @return array + * @throws InvalidVideoException when the bytes cannot be a video + */ + public function store(string $bytes, string $originalName, string $folder): array + { + $in = fopen('php://memory', 'r+b'); + fwrite($in, $bytes); + rewind($in); + try { + return $this->storeStream($in, strlen($bytes), $originalName, $folder); + } finally { + fclose($in); + } + } + + /** + * Like store(), straight from the request: the bytes go to disk as they come, and only their + * first ones and their hash are looked at, so a video of 2 GB needs no 2 GB of memory. + * + * @param resource $in + * @return array + * @throws VideoTooLargeException when the stream has more than `$maxBytes` + * @throws InvalidVideoException when the bytes cannot be a video + */ + public function storeStream($in, int $maxBytes, string $originalName, string $folder): array + { + $upload = $this->queue->receive($in, $maxBytes); + try { + if (!self::looksLikeVideo((string) file_get_contents($upload, false, null, 0, 256))) { + throw new InvalidVideoException('Not a video file – use ' . self::SIGNATURE_HINT); + } + if (!preg_match('/^' . UserStore::FOLDER_PATTERN . '$/', $folder)) { + throw new InvalidArgumentException("Invalid folder: {$folder}"); + } + $hash = hash_file('sha256', $upload); + $dir = "{$this->root}/{$folder}"; + if (!is_dir($dir) && !mkdir($dir, 0775, true) && !is_dir($dir)) { + throw new RuntimeException("Cannot create directory {$dir}"); + } + OriginalName::write($dir, "{$hash}.mp4", $originalName); + if (!is_file("{$dir}/{$hash}.mp4")) { + $job = $this->queue->job($folder, $hash); + if ($job === null || $job['state'] === VideoQueue::FAILED) { + $this->queue->add($folder, $hash, $upload, OriginalName::sanitize($originalName)); + } + } + return $this->record("{$folder}/{$hash}.mp4"); + } finally { + if (is_file($upload)) { + unlink($upload); + } + } + } + + /** + * One file as the picker shows it, done or not; null when there is neither file nor job. + * + * @return array|null + */ + public function record(string $file): ?array + { + if (!self::isValidName($file)) { + return null; + } + if (is_file("{$this->root}/{$file}")) { + $folder = self::folderOf($file); + $dir = $folder === '' ? $this->root : "{$this->root}/{$folder}"; + return [ + 'file' => $file, + 'name' => OriginalName::read($dir, basename($file)), + ...$this->details("{$this->root}/{$file}"), + 'bytes' => (int) filesize("{$this->root}/{$file}"), + ]; + } + $job = $this->jobOf($file); + return $job === null ? null : $this->pending($job); + } + + /** A video is there for deleting while it is converted or failed, too. */ + public function exists(string $file): bool + { + return parent::exists($file) || $this->jobOf($file) !== null; + } + + /** Lists the converted files like every store does, and the ones on their way after them. */ + public function list(string $folder, bool $seesAll, array $owners): array + { + $items = parent::list($folder, $seesAll, $owners); + foreach ($this->queue->jobs() as $job) { + if ($seesAll || $job['folder'] === $folder) { + $items[] = $this->pending($job) + [ + 'updatedAt' => (string) $job['createdAt'], + 'owner' => $owners[$job['folder']] ?? '', + 'deletable' => true, + ]; + } + } + return $items; + } + + public function delete(string $file): bool + { + if (!self::isValidName($file)) { + return false; + } + $folder = self::folderOf($file); + $hash = basename($file, '.mp4'); + $removed = $folder !== '' && $this->queue->remove($folder, $hash); + if (is_file("{$this->root}/{$file}")) { + $removed = parent::delete($file) || $removed; + $dir = $folder === '' ? $this->root : "{$this->root}/{$folder}"; + foreach (["{$dir}/{$hash}.jpg", "{$dir}/{$hash}.meta.json"] as $path) { + if (is_file($path)) { + unlink($path); + } + } + } elseif ($removed) { + OriginalName::delete("{$this->root}/{$folder}", "{$hash}.mp4"); + } + return $removed; + } + + /** Called by the worker once the MP4 is in place: what the editor reads instead of probing. */ + public static function writeDetails(string $dir, string $hash, array $details): void + { + $json = json_encode($details, JSON_UNESCAPED_SLASHES); + AtomicFile::write("{$dir}/{$hash}.meta.json", (string) $json, AtomicFile::SERVED, 'video details'); + } + + /** @return array */ + protected function details(string $path): array + { + $meta = substr($path, 0, -strlen('.mp4')) . '.meta.json'; + $raw = is_file($meta) ? file_get_contents($meta) : false; + $details = $raw === false ? null : json_decode($raw, true); + $details = is_array($details) ? $details : []; + $file = substr($path, strlen($this->root) + 1); + return [ + 'status' => 'ready', + 'width' => (int) ($details['width'] ?? 0), + 'height' => (int) ($details['height'] ?? 0), + 'duration' => (float) ($details['duration'] ?? 0), + 'fps' => (float) ($details['fps'] ?? 0), + 'audio' => (bool) ($details['audio'] ?? false), + 'poster' => substr($file, 0, -strlen('.mp4')) . '.jpg', + ]; + } + + /** @return array|null */ + private function jobOf(string $file): ?array + { + $folder = self::folderOf($file); + return $folder === '' ? null : $this->queue->job($folder, basename($file, '.mp4')); + } + + /** + * @param array $job + * @return array + */ + private function pending(array $job): array + { + return [ + 'file' => "{$job['folder']}/{$job['hash']}.mp4", + 'name' => (string) $job['name'], + 'status' => (string) $job['state'], + 'progress' => (float) $job['progress'], + 'error' => (string) $job['error'], + 'bytes' => 0, + ]; + } + + private static function looksLikeVideo(string $bytes): bool + { + if (strlen($bytes) < 16) { + return false; + } + return substr($bytes, 4, 4) === 'ftyp' + || str_starts_with($bytes, "\x1A\x45\xDF\xA3") + || (str_starts_with($bytes, 'RIFF') && substr($bytes, 8, 4) === 'AVI ') + || ($bytes[0] === "\x47" && strlen($bytes) > 188 && $bytes[188] === "\x47"); + } +} + +/** Thrown by VideoStore::store when the uploaded bytes cannot be a video. */ +final class InvalidVideoException extends RuntimeException +{ +} + +/** Thrown by VideoQueue::receive when an upload is bigger than the API takes. */ +final class VideoTooLargeException extends RuntimeException +{ +} diff --git a/api/lib/VideoWorker.php b/api/lib/VideoWorker.php new file mode 100644 index 00000000..70d2b0c2 --- /dev/null +++ b/api/lib/VideoWorker.php @@ -0,0 +1,150 @@ +store->queue()->next(); + if ($job === null) { + return false; + } + $this->convert((string) $job['folder'], (string) $job['hash']); + return true; + } + + private function convert(string $folder, string $hash): void + { + $queue = $this->store->queue(); + if (!$queue->update($folder, $hash, ['state' => VideoQueue::CONVERTING, 'progress' => 0, 'error' => ''])) { + return; + } + $dir = "{$this->videos}/{$folder}"; + $tag = bin2hex(random_bytes(4)); + $tmpVideo = "{$dir}/{$hash}.{$tag}.tmp.mp4"; + $tmpPoster = "{$dir}/{$hash}.{$tag}.tmp.jpg"; + try { + $plan = VideoFormat::plan(self::probe($queue->source($folder, $hash))); + $finished = $this->run( + VideoFormat::convertArgs($queue->source($folder, $hash), $tmpVideo, $plan), + $plan['duration'], + static fn(float $p): bool => $queue->update($folder, $hash, ['progress' => round($p, 3)]), + ); + if (!$finished) { + return; + } + $details = VideoFormat::details(self::probe($tmpVideo)); + $this->run(VideoFormat::posterArgs($tmpVideo, $tmpPoster, $details['duration']), 0, null); + if ($queue->job($folder, $hash) === null) { + return; + } + chmod($tmpVideo, AtomicFile::SERVED); + chmod($tmpPoster, AtomicFile::SERVED); + VideoStore::writeDetails($dir, $hash, $details); + rename($tmpPoster, "{$dir}/{$hash}.jpg"); + rename($tmpVideo, "{$dir}/{$hash}.mp4"); + $queue->remove($folder, $hash); + } catch (Throwable $e) { + $queue->dropSource($folder, $hash); + $queue->update($folder, $hash, ['state' => VideoQueue::FAILED, 'error' => self::shorten($e->getMessage())]); + } finally { + foreach ([$tmpVideo, $tmpPoster] as $path) { + if (is_file($path)) { + unlink($path); + } + } + } + } + + /** @return array */ + private static function probe(string $path): array + { + [$code, $out, $err] = self::capture(VideoFormat::probeArgs($path)); + $probe = json_decode($out, true); + if ($code !== 0 || !is_array($probe)) { + throw new RuntimeException('ffmpeg cannot read this file' . ($err === '' ? '' : ": {$err}")); + } + return $probe; + } + + /** + * Runs ffmpeg, reporting progress from its `-progress pipe:1` lines to `$report`. Returns false + * when `$report` says the job is gone, after stopping ffmpeg. + * + * @param list $args + * @param (callable(float):bool)|null $report + */ + private function run(array $args, float $duration, ?callable $report): bool + { + $err = tempnam(sys_get_temp_dir(), 'video-err-'); + $process = proc_open($args, [0 => ['file', '/dev/null', 'r'], 1 => ['pipe', 'w'], 2 => ['file', $err, 'w']], $pipes); + if (!is_resource($process)) { + throw new RuntimeException('Cannot start ffmpeg'); + } + $last = 0.0; + $alive = true; + while (($line = fgets($pipes[1])) !== false) { + $share = $report === null ? null : VideoFormat::progress($line, $duration); + if ($share !== null && microtime(true) - $last >= self::PROGRESS_EVERY) { + $last = microtime(true); + if (!$report($share)) { + $alive = false; + proc_terminate($process); + break; + } + } + } + fclose($pipes[1]); + $code = proc_close($process); + $message = trim((string) file_get_contents($err)); + unlink($err); + if (!$alive) { + return false; + } + if ($code !== 0) { + throw new RuntimeException($message === '' ? "ffmpeg stopped with code {$code}" : $message); + } + return true; + } + + /** + * @param list $args + * @return array{0:int,1:string,2:string} + */ + private static function capture(array $args): array + { + $process = proc_open($args, [0 => ['file', '/dev/null', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + if (!is_resource($process)) { + throw new RuntimeException('Cannot start ffprobe'); + } + $out = (string) stream_get_contents($pipes[1]); + $err = trim((string) stream_get_contents($pipes[2])); + fclose($pipes[1]); + fclose($pipes[2]); + return [proc_close($process), $out, $err]; + } + + /** The last lines of what ffmpeg said: enough to tell why, not a log. */ + private static function shorten(string $message): string + { + $lines = array_values(array_filter(array_map('trim', explode("\n", $message)), static fn(string $l): bool => $l !== '')); + $text = implode(' · ', array_slice($lines, -2)); + return mb_strlen($text) > 300 ? mb_substr($text, 0, 300) . '…' : $text; + } +} diff --git a/src/core/commands/assetCommands.ts b/src/core/commands/assetCommands.ts index 58814888..50caae29 100644 --- a/src/core/commands/assetCommands.ts +++ b/src/core/commands/assetCommands.ts @@ -10,6 +10,7 @@ import type { TextureAsset, TilemapLayer, VariablesAsset, + VideoAsset, } from '../model/Asset'; import type { AssetType } from '../model/AssetTypes'; import { embeddedIn } from '../music/embedded'; @@ -33,6 +34,7 @@ import { } from '../model/Scope'; import type { SceneObject } from '../model/SceneObject'; import { isTextureFile } from '../model/texture'; +import { isVideoFile } from '../model/video'; import { cellsWithout } from '../model/tilemap'; import { ASSET_FX_KEY } from '../properties/assetFx'; import { coerceValue, defaultsFor, initialOf, initialPropsFor } from '../properties/PropertySchema'; @@ -108,6 +110,8 @@ export function buildAsset(type: AssetType, name: string, id = newId()): Asset { return { ...base, type: 'font', file: '' }; case 'sound': return { ...base, type: 'sound', file: '' }; + case 'video': + return { ...base, type: 'video', file: '', width: 0, height: 0, duration: 0, fps: 0, audio: false }; case 'soundfx': return { ...base, type: 'soundfx', graph: createEmptyGraph() }; case 'instrument': @@ -311,6 +315,25 @@ export class CreateSoundCommand extends ImportFileCommand { } } +/** What a video says about itself once the worker has made it (#402), copied into the asset. */ +export interface VideoDetails { + width: number; + height: number; + duration: number; + fps: number; + audio: boolean; +} + +export class CreateVideoCommand extends ImportFileCommand { + constructor(name: string, file: string, details: VideoDetails) { + super( + { id: newId(), name, props: defaultsFor('video'), type: 'video', file, ...details }, + 'video', + isVideoFile, + ); + } +} + /** * Makes an asset a song kept to itself a project asset (#355): it shows in the lists, and a name a * listed asset has already gets a number. diff --git a/src/core/commands/clipCommands.ts b/src/core/commands/clipCommands.ts index 72dca76f..c851c096 100644 --- a/src/core/commands/clipCommands.ts +++ b/src/core/commands/clipCommands.ts @@ -1,29 +1,41 @@ -/** The clips of a sound (#357): stretches of its file with a name, as frames are of a texture. */ +/** + * The clips of a sound (#357) or a video (#402): stretches of its file with a name, as frames are + * of a texture. + */ import { newId } from '../ids'; -import type { SoundAsset, SoundClip } from '../model/Asset'; +import type { SoundClip } from '../model/Asset'; import type { PropertyBag, PropertyValue } from '../model/Properties'; import type { Project } from '../model/Project'; -import { clipsOf, nextClipName, uniqueClipName } from '../model/sound'; +import { type ClipOwner, clipsOf, hasClips, nextClipName, uniqueClipName } from '../model/sound'; +import type { PropertyDef } from '../properties/PropertySchema'; import { CLIP_ROWS } from '../properties/sounds'; +import { VIDEO_CLIP_ROWS } from '../properties/videos'; import { type Removed, removeFirst, restoreAt } from '../util/list'; import { type Command, CommandError } from './Command'; import { requireAsset } from './require'; -function requireSound(project: Project, assetId: string): SoundAsset { +/** A clip is the same shape on a sound and on a video; only its rows differ. */ +type Clip = SoundClip; + +function requireSound(project: Project, assetId: string): ClipOwner { const asset = requireAsset(project, assetId); - if (asset.type !== 'sound') throw new CommandError(`${asset.name} is not a sound`); + if (!hasClips(asset)) throw new CommandError(`${asset.name} has no clips`); return asset; } -function requireClip(asset: SoundAsset, clipId: string): SoundClip { - const clip = clipsOf(asset).find((c) => c.id === clipId); +function requireClip(asset: ClipOwner, clipId: string): Clip { + const clip = clipsOf(asset).find((c) => c.id === clipId); if (!clip) throw new CommandError(`Clip ${clipId} not found in ${asset.name}`); return clip; } -/** What a clip may hold: a row of `CLIP_ROWS`, a finite number at least 0, a curve it offers. */ -function heldClipValue(key: string, value: PropertyValue): PropertyValue { - const def = CLIP_ROWS.find((row) => row.key === key); +function clipRowsOf(asset: ClipOwner): readonly PropertyDef[] { + return asset.type === 'video' ? VIDEO_CLIP_ROWS : CLIP_ROWS; +} + +/** What a clip may hold: a row of its asset's clip rows, a finite number at least 0, a curve it offers. */ +function heldClipValue(asset: ClipOwner, key: string, value: PropertyValue): PropertyValue { + const def = clipRowsOf(asset).find((row) => row.key === key); if (!def) throw new CommandError(`A clip has no "${key}"`); if (def.kind === 'number' && (typeof value !== 'number' || !Number.isFinite(value) || value < 0)) throw new CommandError(`${def.label} needs a number of at least 0`); @@ -47,14 +59,14 @@ export class AddClipCommand implements Command { execute(project: Project): void { const asset = requireSound(project, this.assetId); const props: PropertyBag = {}; - for (const [key, value] of Object.entries(this.props)) props[key] = heldClipValue(key, value); + for (const [key, value] of Object.entries(this.props)) props[key] = heldClipValue(asset, key, value); // Named per execution: a redo has to fit the names that exist when it runs. - asset.clips = [...clipsOf(asset), { id: this.clipId, name: nextClipName(asset), props }]; + asset.clips = [...clipsOf(asset), { id: this.clipId, name: nextClipName(asset), props }]; } undo(project: Project): void { const asset = requireSound(project, this.assetId); - asset.clips = clipsOf(asset).filter((c) => c.id !== this.clipId); + asset.clips = clipsOf(asset).filter((c) => c.id !== this.clipId); } } @@ -73,8 +85,11 @@ export class SetClipPropsCommand implements Command { ) {} execute(project: Project): void { - const clip = requireClip(requireSound(project, this.assetId), this.clipId); - const held = Object.entries(this.values).map(([key, value]) => [key, heldClipValue(key, value)] as const); + const asset = requireSound(project, this.assetId); + const clip = requireClip(asset, this.clipId); + const held = Object.entries(this.values).map( + ([key, value]) => [key, heldClipValue(asset, key, value)] as const, + ); this.previous = held.map(([key]) => [key, clip.props[key]]); for (const [key, value] of held) clip.props[key] = value; } @@ -117,7 +132,7 @@ export class RenameClipCommand implements Command { */ export class RemoveClipCommand implements Command { readonly label = 'Delete clip'; - private removed?: Removed; + private removed?: Removed; constructor( private readonly assetId: string, @@ -126,7 +141,7 @@ export class RemoveClipCommand implements Command { execute(project: Project): void { const asset = requireSound(project, this.assetId); - const clips = [...clipsOf(asset)]; + const clips = [...clipsOf(asset)]; const removed = removeFirst(clips, (c) => c.id === this.clipId); if (!removed) throw new CommandError(`Clip ${this.clipId} not found in ${asset.name}`); this.removed = removed; @@ -136,7 +151,7 @@ export class RemoveClipCommand implements Command { undo(project: Project): void { if (!this.removed) return; const asset = requireSound(project, this.assetId); - const clips = [...clipsOf(asset)]; + const clips = [...clipsOf(asset)]; restoreAt(clips, [this.removed]); asset.clips = clips; } diff --git a/src/core/model/Asset.ts b/src/core/model/Asset.ts index 302c071f..1ff654c1 100644 --- a/src/core/model/Asset.ts +++ b/src/core/model/Asset.ts @@ -348,6 +348,31 @@ export interface SoundClip { stream?: SoundStream; } +/** + * `file` names the video in `data/videos/` (#402): `{folder}/{sha256}.mp4`, the MP4 the worker made + * from the upload, with a poster `.jpg` and its details beside it. The size, length, frame rate and + * whether it has sound are copied in when it is picked, so a sprite can be measured before the file + * loads. A sprite wears it through `texture`; its clips are stretches of it, as a sound's are. + */ +export interface VideoAsset extends AssetBase { + type: 'video'; + file: string; + width: number; + height: number; + duration: number; + fps: number; + audio: boolean; + /** Absent for a video without clips. */ + clips?: VideoClip[]; +} + +/** A stretch of a video with a name, `props` its start and end (see `properties/videos.ts`). */ +export interface VideoClip { + id: string; + name: string; + props: PropertyBag; +} + /** * What a sound is played through (#311): the graph runs from its `Input` over effect nodes to its * `Output`, and a `Play sound` naming it sends the voice along that line (see `core/soundfx`). @@ -658,7 +683,8 @@ export type Asset = | PatternAsset | SongAsset | BusAsset - | ParticlesAsset; + | ParticlesAsset + | VideoAsset; /** The assets whose `variables` are declared by the user – see `requireVariableOwner`. */ export type VariableOwnerAsset = BehaviorAsset | VariablesAsset | StructAsset; diff --git a/src/core/model/AssetTypes.ts b/src/core/model/AssetTypes.ts index 6db62ca2..f826bf61 100644 --- a/src/core/model/AssetTypes.ts +++ b/src/core/model/AssetTypes.ts @@ -30,6 +30,7 @@ export const ASSET_TYPES = [ 'song', 'bus', 'particles', + 'video', ] as const; export type AssetType = (typeof ASSET_TYPES)[number]; @@ -66,6 +67,7 @@ export const ASSET_TYPE_LABELS: Record = { song: 'Song', bus: 'Bus', particles: 'Particles', + video: 'Video', }; /** @@ -79,7 +81,8 @@ export const ASSET_TYPE_LABELS: Record = { * A shader function is part of other shaders (#303), worn by nothing itself. A sound FX is what a * sound is played through (#311), heard but never seen, an instrument what a note is played * with (#331), a pattern the notes it plays (#333) and a song what lays them out (#334). A bus is - * where sounds are mixed on their way out (#325). + * where sounds are mixed on their way out (#325). A video is a moving picture a sprite wears through + * `texture` (#402), as it wears an image. */ export type PlaceableAssetType = Exclude< AssetType, @@ -103,6 +106,7 @@ export type PlaceableAssetType = Exclude< | 'bus' | 'variables' | 'struct' + | 'video' >; const NOT_PLACEABLE: ReadonlySet = new Set([ @@ -126,6 +130,7 @@ const NOT_PLACEABLE: ReadonlySet = new Set([ 'bus', 'variables', 'struct', + 'video', ]); export function isPlaceableType(type: AssetType): type is PlaceableAssetType { diff --git a/src/core/model/sound.ts b/src/core/model/sound.ts index ccfc745c..4ae98835 100644 --- a/src/core/model/sound.ts +++ b/src/core/model/sound.ts @@ -1,6 +1,6 @@ import { nextFreeName, nextNumberedName } from '../commands/naming'; import { newId } from '../ids'; -import type { Asset, SoundAsset, SoundClip } from './Asset'; +import type { Asset, SoundAsset, SoundClip, VideoAsset } from './Asset'; import { findAsset, type Project } from './Project'; import { storedFilePattern } from './storedFile'; @@ -13,10 +13,18 @@ export function isSoundFile(name: string): boolean { const CLIP_SEPARATOR = ' / '; const DEFAULT_CLIP_NAME = 'Clip'; -export function clipsOf(asset: SoundAsset): readonly SoundClip[] { +/** The clips of a sound (#357) or a video (#402). */ +export function clipsOf(asset: { clips?: C[] }): readonly C[] { return asset.clips ?? []; } +/** What has clips: a sound and a video, each a file with named stretches of it. */ +export type ClipOwner = SoundAsset | VideoAsset; + +export function hasClips(asset: Asset): asset is ClipOwner { + return asset.type === 'sound' || asset.type === 'video'; +} + /** What a sound field names (#357): a sound, the whole file, or one of its clips. */ export interface SoundRef { asset: SoundAsset; @@ -39,10 +47,12 @@ export function findSound(project: Project, id: string): SoundRef | null { return null; } -/** The asset among `assets` an id names: the asset itself, or the sound one of whose clips it is. */ +/** The asset among `assets` an id names: the asset itself, or the one one of whose clips it is. */ export function assetNamedBy(assets: readonly Asset[], id: string): Asset | undefined { if (id === '') return undefined; - return assets.find((a) => a.id === id || (a.type === 'sound' && clipsOf(a).some((c) => c.id === id))); + return assets.find( + (a) => a.id === id || (hasClips(a) && clipsOf<{ id: string }>(a).some((c) => c.id === id)), + ); } /** @@ -50,8 +60,8 @@ export function assetNamedBy(assets: readonly Asset[], id: string): Asset | unde * each old id with its new one in `ids` so the places that named a clip can follow. */ export function renewClipIds(copy: Asset, ids: Map): void { - if (copy.type !== 'sound') return; - for (const clip of clipsOf(copy)) { + if (!hasClips(copy)) return; + for (const clip of clipsOf<{ id: string }>(copy)) { const id = newId(); ids.set(clip.id, id); clip.id = id; @@ -63,18 +73,18 @@ export function soundRefName(ref: SoundRef, soundName = ref.asset.name): string return ref.clip ? `${soundName}${CLIP_SEPARATOR}${ref.clip.name}` : soundName; } -/** "Clip 1", 2, 3 …: the first number none of the sound's clips has. */ -export function nextClipName(asset: SoundAsset): string { +/** "Clip 1", 2, 3 …: the first number none of the asset's clips has. */ +export function nextClipName(asset: ClipOwner): string { return nextNumberedName( - clipsOf(asset).map((c) => c.name), + clipsOf<{ name: string }>(asset).map((c) => c.name), DEFAULT_CLIP_NAME, ); } -/** A name free among the sound's clips: `name`, else "name 2", "name 3" … */ -export function uniqueClipName(asset: SoundAsset, name: string, ignoreId = ''): string { +/** A name free among the asset's clips: `name`, else "name 2", "name 3" … */ +export function uniqueClipName(asset: ClipOwner, name: string, ignoreId = ''): string { return nextFreeName( - clipsOf(asset) + clipsOf<{ id: string; name: string }>(asset) .filter((c) => c.id !== ignoreId) .map((c) => c.name), name, diff --git a/src/core/model/texture.ts b/src/core/model/texture.ts index eb732d4d..02cf36b0 100644 --- a/src/core/model/texture.ts +++ b/src/core/model/texture.ts @@ -4,6 +4,7 @@ import { findPixelFrame, pixelFrameRect } from './pixelart'; import { type PropertyBag, textOf, type Vec2 } from './Properties'; import { findAsset, type Project } from './Project'; import { storedFilePattern } from './storedFile'; +import { findVideo, isVideoFile, videoKey, videoPlayOf, type VideoRef } from './video'; const FILE_PATTERN = storedFilePattern('png|jpg|webp|gif'); @@ -95,13 +96,17 @@ export function resolveTexture(project: Project, props: PropertyBag): TextureRef return resolvePicture(project, textOf(props.texture ?? '')); } -/** A texture, a frame or a generating asset; any other asset's id is no picture. */ +/** A texture, a frame, a video or a clip of one, or a generating asset; any other asset's id is no picture. */ export function resolvePicture(project: Project, id: string): TextureRef | null { if (!id) return null; const asset = findAsset(project, id); - if (asset) return asset.type === 'texture' ? refFor(asset, null) : generatedRef(asset, project); + if (asset?.type === 'texture') return refFor(asset, null); + if (asset?.type === 'video') return videoRef({ asset, clip: null }); + if (asset) return generatedRef(asset, project); const found = findFrame(project, id); if (found) return refFor(found.asset, found.frame); + const video = findVideo(project, id); + if (video) return videoRef(video); // A frame of pixel art (#130) is a cut of its generated sheet, as a texture frame is of its image. const pixel = findPixelFrame(project, id); return pixel ? generatedRef(pixel.asset, project, pixelFrameRect(pixel.asset, pixel.index)) : null; @@ -128,6 +133,19 @@ function generatedRef(asset: Asset, project: Project, rect: FrameRect | null = n }; } +/** A playing video (#402) reads like a texture of its size; the key says what plays, see `videoKey`. */ +function videoRef(ref: VideoRef): TextureRef | null { + if (!isVideoFile(ref.asset.file)) return null; + return { + file: videoKey(videoPlayOf(ref)), + smooth: true, + rect: null, + sourceWidth: ref.asset.width, + sourceHeight: ref.asset.height, + slice: null, + }; +} + function refFor(asset: TextureAsset, frame: TextureFrame | null): TextureRef | null { if (!isTextureFile(asset.file)) return null; return { diff --git a/src/core/model/video.ts b/src/core/model/video.ts new file mode 100644 index 00000000..0076eaff --- /dev/null +++ b/src/core/model/video.ts @@ -0,0 +1,110 @@ +import { videoLoopsOf, videoSpanOf, videoVolumeOf } from '../properties/videos'; +import type { VideoAsset, VideoClip } from './Asset'; +import { findAsset, type Project } from './Project'; +import { clipsOf } from './sound'; +import { storedFilePattern } from './storedFile'; + +const FILE_PATTERN = storedFilePattern('mp4'); + +export function isVideoFile(name: string): boolean { + return FILE_PATTERN.test(name); +} + +/** The still the worker cut from a video (#402), lying beside it. */ +export function posterFileOf(file: string): string { + return file.replace(/\.mp4$/, '.jpg'); +} + +/** What a texture field names when it names a video: the whole video, or one of its clips. */ +export interface VideoRef { + asset: VideoAsset; + clip: VideoClip | null; +} + +/** A video or a clip of one, by id; clip ids share the generator with asset ids, as frames do. */ +export function findVideo(project: Project, id: string): VideoRef | null { + if (id === '') return null; + const asset = findAsset(project, id); + if (asset) return asset.type === 'video' ? { asset, clip: null } : null; + for (const video of project.assets) { + if (video.type !== 'video') continue; + const clip = clipsOf(video).find((c) => c.id === id); + if (clip) return { asset: video, clip }; + } + return null; +} + +/** Every id that means "this video": the asset itself and each of its clips. */ +export function videoRefIds(asset: VideoAsset): string[] { + return [asset.id, ...clipsOf(asset).map((c) => c.id)]; +} + +/** + * The stretch a clip plays, inside the video: a start before its end, an end of 0 or past the + * video's length being its end. Without a length (not known yet) the asked values stand. + */ +export function videoSpan(duration: number, start: number, end: number): { start: number; end: number } { + const length = duration > 0 ? duration : Infinity; + const from = Math.min(Math.max(0, start), Number.isFinite(length) ? Math.max(0, length - 0.001) : start); + const to = end > 0 ? Math.min(end, length) : length; + return { start: from, end: to > from ? to : length }; +} + +/** How a video is played: which file, which stretch of it, whether it repeats, how loud. */ +export interface VideoPlay { + file: string; + start: number; + /** Seconds; Infinity for the end of the video, wherever that is. */ + end: number; + loop: boolean; + volume: number; +} + +export function videoPlayOf(ref: VideoRef): VideoPlay { + const asked = ref.clip ? videoSpanOf(ref.clip.props) : { start: 0, end: 0 }; + const span = videoSpan(ref.asset.duration, asked.start, asked.end); + return { + file: ref.asset.file, + start: span.start, + end: ref.asset.duration > 0 && span.end >= ref.asset.duration ? Infinity : span.end, + loop: videoLoopsOf(ref.asset.props), + volume: videoVolumeOf(ref.asset.props), + }; +} + +const PREFIX = 'video:'; + +/** + * The name a playing video goes by where a picture is named by `TextureRef.file` (see + * `generated.ts`): `video:#~~~`. No uploaded image looks like it, + * and one key is one player: two sprites wearing the same clip show the same picture. + */ +export function videoKey(play: VideoPlay): string { + const end = Number.isFinite(play.end) ? play.end : 0; + return `${PREFIX}${play.file}#${play.start}~${end}~${play.loop ? 1 : 0}~${play.volume}`; +} + +export function isVideoKey(key: string): boolean { + return parseVideoKey(key) !== null; +} + +export function parseVideoKey(key: string): VideoPlay | null { + if (!key.startsWith(PREFIX)) return null; + const hash = key.indexOf('#'); + if (hash < 0) return null; + const file = key.slice(PREFIX.length, hash); + const parts = key.slice(hash + 1).split('~'); + const [start, end, loop, volume] = parts.map(Number); + if (!isVideoFile(file) || parts.length !== 4 || ![start, end, loop, volume].every(Number.isFinite)) + return null; + return { file, start: start!, end: end! > 0 ? end! : Infinity, loop: loop === 1, volume: volume! }; +} + +/** Where in its file a player is after `elapsed` seconds of playing (#402). */ +export function videoTimeAt(play: Pick, elapsed: number): number { + const length = play.end - play.start; + const run = Math.max(0, elapsed); + if (!Number.isFinite(length) || length <= 0) return play.start + run; + if (!play.loop) return play.start + Math.min(run, length); + return play.start + (run % length); +} diff --git a/src/core/properties/PropertySchema.ts b/src/core/properties/PropertySchema.ts index 87f94de9..599699f4 100644 --- a/src/core/properties/PropertySchema.ts +++ b/src/core/properties/PropertySchema.ts @@ -129,6 +129,7 @@ import { import { DEFAULT_SIGN, DEFAULT_TEMPO, MAX_TEMPO, MIN_TEMPO, SIGN_OPTIONS } from '../music/time'; import { ASSET_FX_KEY } from './assetFx'; import { BUS_FX_KEY, BUS_VOLUME_KEY } from './buses'; +import { VIDEO_ROWS } from './videos'; import { Theme } from '../../ui'; /** @@ -138,7 +139,7 @@ import { Theme } from '../../ui'; * `assetTypes` an asset picker offers every placeable asset; `none` is what it calls an empty pick, * `master` offers the master bus as a choice of its own, `noFx` "None" as one where empty means * "from the asset" (#352); `uploadedOnly` keeps drawn pictures out - * of a texture picker (#186). `renderLayers` makes a layer row pick from the project's render layers + * of a texture picker (#186), `moving` lets videos and their clips in (#402). `renderLayers` makes a layer row pick from the project's render layers * instead of its collision layers (#302), with "Inherit" as the first choice of a single layer. */ type Extras = K extends 'number' @@ -153,7 +154,7 @@ type Extras = K extends 'number' noFx?: boolean; } : K extends 'texture' - ? { uploadedOnly?: boolean } + ? { uploadedOnly?: boolean; moving?: boolean } : K extends 'layer' | 'layers' ? { renderLayers?: boolean } : unknown; @@ -529,7 +530,8 @@ const ASSET_SCHEMAS: Record = { { key: 'height', label: 'Height', kind: 'number', default: 100, min: 1, step: 1 }, // The fill while there is no picture, and a tint multiplier once there is one: white = no tint. { key: 'color', label: 'Color', kind: 'color', default: 0xffffff }, - { key: 'texture', label: 'Texture', kind: 'texture', default: '' }, + // A video or a clip of one plays on the sprite (#402). + { key: 'texture', label: 'Texture', kind: 'texture', default: '', moving: true }, // The shader's own parameters follow this row – see `schemaForAsset`. Empty is no shader at all. { key: SHADER_KEY, label: 'Shader', kind: 'asset', default: '', assetTypes: ['shader'] }, ], @@ -760,6 +762,7 @@ const ASSET_SCHEMAS: Record = { ], // Where sounds are mixed (#325): how loud and through which sound FX. Where it goes on is the // master routing's (#351). + video: [...VIDEO_ROWS], bus: [ { key: BUS_VOLUME_KEY, label: 'Volume', kind: 'number', default: 1, min: 0, max: 1, step: 0.05 }, { key: BUS_FX_KEY, label: 'Sound FX', kind: 'asset', default: '', assetTypes: ['soundfx'] }, diff --git a/src/core/properties/textureRef.ts b/src/core/properties/textureRef.ts index 9a23a211..7bb9bb83 100644 --- a/src/core/properties/textureRef.ts +++ b/src/core/properties/textureRef.ts @@ -1,8 +1,10 @@ -import type { Asset, PixelFrame, TextureFrame } from '../model/Asset'; +import type { Asset, PixelFrame, TextureFrame, VideoClip } from '../model/Asset'; import { generatesPictures } from '../model/generated'; import { findPixelFrame, frameLabel } from '../model/pixelart'; import { findAsset, type Project } from '../model/Project'; +import { clipsOf } from '../model/sound'; import { findFrame } from '../model/texture'; +import { findVideo } from '../model/video'; export const NO_TEXTURE_LABEL = 'None'; @@ -14,13 +16,13 @@ export interface TextureChoice { label: string; /** The texture, or the asset that draws its own picture (#185). */ asset: Asset; - /** Null for the whole-picture entry. */ - frame: TextureFrame | PixelFrame | null; + /** Null for the whole-picture entry; a clip for one of a video (#402). */ + frame: TextureFrame | PixelFrame | VideoClip | null; } -/** An asset a texture field may name: an uploaded picture or a drawn one. */ +/** An asset a texture field may name: an uploaded picture, a drawn one, or a video (#402). */ function isPictureAsset(asset: Asset): boolean { - return asset.type === 'texture' || generatesPictures(asset.type); + return asset.type === 'texture' || asset.type === 'video' || generatesPictures(asset.type); } /** @@ -36,7 +38,25 @@ export function textureRefLabel(project: Project | null, id: string): string { const found = findFrame(project, id); if (found) return `${found.asset.name}${FRAME_SEPARATOR}${found.frame.name}`; const pixel = findPixelFrame(project, id); - return pixel ? `${pixel.asset.name}${FRAME_SEPARATOR}${frameLabel(pixel.index)}` : `${id}?`; + if (pixel) return `${pixel.asset.name}${FRAME_SEPARATOR}${frameLabel(pixel.index)}`; + const clip = findVideo(project, id); + return clip?.clip ? `${clip.asset.name}${FRAME_SEPARATOR}${clip.clip.name}` : `${id}?`; +} + +/** The videos and their clips (#402), what a texture field that plays them offers after the pictures. */ +export function videoChoices(project: Project): TextureChoice[] { + const videos = project.assets + .filter((a) => a.type === 'video') + .sort((a, b) => a.name.localeCompare(b.name)); + return videos.flatMap((asset) => [ + { id: asset.id, label: asset.name, asset, frame: null }, + ...clipsOf(asset).map((clip) => ({ + id: clip.id, + label: `${asset.name}${FRAME_SEPARATOR}${clip.name}`, + asset, + frame: clip, + })), + ]); } /** @@ -45,7 +65,9 @@ export function textureRefLabel(project: Project | null, id: string): string { * not depend on upload time; frames keep creation order, which for a sliced sheet is reading order. */ export function pictureChoices(project: Project): TextureChoice[] { - const pictures = project.assets.filter(isPictureAsset).sort((a, b) => a.name.localeCompare(b.name)); + const pictures = project.assets + .filter((asset) => asset.type !== 'video' && isPictureAsset(asset)) + .sort((a, b) => a.name.localeCompare(b.name)); const choices: TextureChoice[] = []; for (const asset of pictures) { choices.push({ id: asset.id, label: asset.name, asset, frame: null }); diff --git a/src/core/properties/videos.ts b/src/core/properties/videos.ts new file mode 100644 index 00000000..ac0d7d38 --- /dev/null +++ b/src/core/properties/videos.ts @@ -0,0 +1,36 @@ +import { booleanOf, type PropertyBag } from '../model/Properties'; +import { coerceAs } from './kinds'; +import type { PropertyDef } from './PropertySchema'; + +/** How loud the video's own sound is, 0–1 (#402). */ +export const VIDEO_VOLUME_KEY = 'volume'; +/** Whether it starts over at its end – a background loop is what a placed video mostly is. */ +export const VIDEO_LOOP_KEY = 'loop'; +/** Where a clip starts and ends, in seconds; an end of 0 is the end of the video. */ +export const VIDEO_START_KEY = 'start'; +export const VIDEO_END_KEY = 'end'; + +export const VIDEO_ROWS: readonly PropertyDef[] = [ + { key: VIDEO_VOLUME_KEY, label: 'Volume', kind: 'number', default: 1, min: 0, max: 1, step: 0.05 }, + { key: VIDEO_LOOP_KEY, label: 'Loop', kind: 'boolean', default: true }, +]; + +/** The rows of a clip of a video, set in the Video tab. */ +export const VIDEO_CLIP_ROWS: readonly PropertyDef[] = [ + { key: VIDEO_START_KEY, label: 'Start (s)', kind: 'number', default: 0, min: 0, step: 0.01, decimals: 3 }, + { key: VIDEO_END_KEY, label: 'End (s)', kind: 'number', default: 0, min: 0, step: 0.01, decimals: 3 }, +]; + +export function videoVolumeOf(bag: PropertyBag): number { + return Math.min(1, Math.max(0, coerceAs(bag[VIDEO_VOLUME_KEY], 'number', 1))); +} + +export function videoLoopsOf(bag: PropertyBag): boolean { + return bag[VIDEO_LOOP_KEY] === undefined ? true : booleanOf(bag[VIDEO_LOOP_KEY]); +} + +/** Start and end of a clip in seconds, `end` 0 for the end of the video. */ +export function videoSpanOf(bag: PropertyBag): { start: number; end: number } { + const seconds = (key: string) => Math.max(0, coerceAs(bag[key], 'number', 0)); + return { start: seconds(VIDEO_START_KEY), end: seconds(VIDEO_END_KEY) }; +} diff --git a/src/core/store/HttpFileStore.ts b/src/core/store/HttpFileStore.ts index d4ac072f..371af9bc 100644 --- a/src/core/store/HttpFileStore.ts +++ b/src/core/store/HttpFileStore.ts @@ -2,10 +2,11 @@ import type { FileStore, StoredFile, UploadedFile } from './FileStore'; import { apiRequest } from './http'; /** - * The API's file collections, one route each; textures, fonts and sounds differ in nothing else. + * The API's file collections, one route each; textures, fonts, sounds and videos differ in nothing + * else. * Pixels (#130) are only ever uploaded – a cel is found by the name in the project, never listed. */ -export type FileRoute = 'textures' | 'fonts' | 'sounds' | 'pixels'; +export type FileRoute = 'textures' | 'fonts' | 'sounds' | 'videos' | 'pixels'; export class HttpFileStore< U extends UploadedFile = UploadedFile, diff --git a/src/core/store/VideoStore.ts b/src/core/store/VideoStore.ts new file mode 100644 index 00000000..11d6bf50 --- /dev/null +++ b/src/core/store/VideoStore.ts @@ -0,0 +1,28 @@ +import type { FileStore, StoredFile, UploadedFile } from './FileStore'; + +/** + * Where an upload is on its way to `data/videos/` (#402): the worker converts one after the other, + * so a video waits, is converted, and is ready or failed. + */ +export type VideoStatus = 'queued' | 'converting' | 'ready' | 'failed'; + +/** + * What the server says about a video. `file` is known the moment the upload is in, long before the + * MP4 exists; the details and the poster come with `ready`, `progress` (0–1) with `converting` + * and `error` with `failed`. + */ +export interface UploadedVideo extends UploadedFile { + status: VideoStatus; + progress?: number; + error?: string; + width?: number; + height?: number; + duration?: number; + fps?: number; + audio?: boolean; + /** `{folder}/{sha256}.jpg`, beside the MP4. */ + poster?: string; +} + +export type StoredVideo = StoredFile & UploadedVideo; +export type VideoStore = FileStore; diff --git a/src/core/store/serialize.ts b/src/core/store/serialize.ts index 24221785..f625756a 100644 --- a/src/core/store/serialize.ts +++ b/src/core/store/serialize.ts @@ -69,6 +69,7 @@ import { decodeCells, encodeCells } from './tilemapCells'; import { isAssetType } from '../model/AssetTypes'; import { isFontFile } from '../model/font'; import { isSoundFile } from '../model/sound'; +import { isVideoFile } from '../model/video'; import { emptyEdit, type InstanceEdit, isEmptyEdit } from '../model/instance'; import { type PropertyBag, type PropertyValue, tagOf, type Vec2 } from '../model/Properties'; import { PROJECT_FORMAT_VERSION, type Project } from '../model/Project'; @@ -117,6 +118,7 @@ import { } from '../properties/PropertySchema'; import { FIELD_SHADER_KEY, particleParamDefs } from '../properties/particles'; import { CLIP_ROWS } from '../properties/sounds'; +import { VIDEO_CLIP_ROWS } from '../properties/videos'; import { SHADER_KEY, shaderParamDefs } from '../properties/shaders'; /** @@ -274,6 +276,23 @@ export function deserializeProject(raw: unknown, fallbackId = ''): Project { ...(clips.length > 0 && { clips }), ...(stream && { stream }), }); + } else if (type === 'video') { + const clips = normalizeClips(a.clips, takenIds, VIDEO_CLIP_ROWS); + const size = (value: unknown) => + typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0; + assets.push({ + id, + type, + name, + props, + file: typeof a.file === 'string' && isVideoFile(a.file) ? a.file : '', + width: Math.round(size(a.width)), + height: Math.round(size(a.height)), + duration: size(a.duration), + fps: size(a.fps), + audio: a.audio === true, + ...(clips.length > 0 && { clips: clips.map(({ id, name, props }) => ({ id, name, props })) }), + }); } else if (type === 'tile') { // A picture naming nothing is kept, like an animation's dangling root: it reads as missing // in the tab and comes back when the texture it named is restored by an undo. @@ -769,8 +788,15 @@ function normalizeTexture( return { file, width, height, frames }; } -/** The clips of a sound (#357); like a frame's, a clip's id may not repeat one the document has. */ -function normalizeClips(raw: unknown, takenIds: Set): SoundClip[] { +/** + * The clips of a sound (#357) or a video (#402); like a frame's, a clip's id may not repeat one the + * document has. + */ +function normalizeClips( + raw: unknown, + takenIds: Set, + rows: readonly PropertyDef[] = CLIP_ROWS, +): SoundClip[] { const clips: SoundClip[] = []; for (const item of Array.isArray(raw) ? raw : []) { const c = asRecord(item); @@ -781,7 +807,7 @@ function normalizeClips(raw: unknown, takenIds: Set): SoundClip[] { clips.push({ id, name: typeof c.name === 'string' ? c.name : '', - props: normalizeBag(c.props, CLIP_ROWS, true), + props: normalizeBag(c.props, rows, true), ...(stream && { stream }), }); } diff --git a/src/editor/App.ts b/src/editor/App.ts index 16d09ad2..e1777293 100644 --- a/src/editor/App.ts +++ b/src/editor/App.ts @@ -7,6 +7,7 @@ import { HttpReleaseStore } from '../core/store/HttpReleaseStore'; import { HttpSnapshotStore } from '../core/store/HttpSnapshotStore'; import { HttpUsersStore } from '../core/store/HttpUsersStore'; import type { StoredTexture, UploadedTexture } from '../core/store/TextureStore'; +import type { StoredVideo, UploadedVideo } from '../core/store/VideoStore'; import { loadPhysics } from '../runtime/physics/engine'; import { projectPictureFactory } from '../viewport/generatedPictures'; import { preloadProjectFiles, setUpProjectFiles } from '../viewport/projectFiles'; @@ -92,6 +93,7 @@ export async function startEditor(app: Application, root: VStack): Promise new HttpFileStore(API_ENDPOINT, 'textures'), new HttpFileStore(API_ENDPOINT, 'fonts'), new HttpFileStore(API_ENDPOINT, 'sounds'), + new HttpFileStore(API_ENDPOINT, 'videos'), new HttpSnapshotStore(API_ENDPOINT), new HttpReleaseStore(API_ENDPOINT), { diff --git a/src/editor/Editor.ts b/src/editor/Editor.ts index ab39c25d..9e460d80 100644 --- a/src/editor/Editor.ts +++ b/src/editor/Editor.ts @@ -9,6 +9,7 @@ import { pluginsOf } from '../core/store/projectPlugins'; import type { FontStore } from '../core/store/FontStore'; import type { SnapshotStore } from '../core/store/SnapshotStore'; import type { SoundStore } from '../core/store/SoundStore'; +import type { VideoStore } from '../core/store/VideoStore'; import { StreamKeeper } from './sound/StreamKeeper'; import type { TextureStore } from '../core/store/TextureStore'; import type { UsersStore } from '../core/store/UsersStore'; @@ -91,6 +92,7 @@ import { PixelArtEditor } from './pixelart/PixelArtEditor'; import { VectorArtEditor } from './vectorart/VectorArtEditor'; import { CurveEditor } from './curve/CurveEditor'; import { SoundEditor } from './sound/SoundEditor'; +import { VideoEditor } from './video/VideoEditor'; import { TextureEditor } from './texture/TextureEditor'; import { installFileImport } from './fileImport'; import type { PluginManager } from './plugins/loader'; @@ -114,6 +116,7 @@ export type CentreTab = | 'pattern' | 'song' | 'game' + | 'video' | 'texture' | 'vectorart' | 'pixelart' @@ -173,6 +176,7 @@ const PANE_ORDER = [ 'instrument', 'pattern', 'song', + 'video', 'texture', 'vectorart', 'pixelart', @@ -201,6 +205,7 @@ const TAB_LABELS: Record = { instrument: 'Instrument', pattern: 'Pattern', song: 'Song', + video: 'Video', texture: 'Texture', vectorart: 'Vector art', pixelart: 'Pixel art', @@ -265,6 +270,7 @@ export class Editor implements PlayHost { readonly shaderEditor: ShaderEditor; readonly particlesEditor: ParticlesEditor; readonly soundEditor: SoundEditor; + readonly videoEditor: VideoEditor; /** Makes the copies streamed sounds play from (#349). */ readonly streamKeeper: StreamKeeper; readonly soundFxEditor: SoundFxEditor; @@ -360,6 +366,7 @@ export class Editor implements PlayHost { textures: TextureStore, fonts: FontStore, sounds: SoundStore, + videos: VideoStore, snapshots: SnapshotStore, releases: ReleaseStore, account: EditorAccount, @@ -373,6 +380,7 @@ export class Editor implements PlayHost { this.particlesEditor = new ParticlesEditor(state); this.streamKeeper = new StreamKeeper(state, sounds); this.soundEditor = new SoundEditor(state, this.streamKeeper); + this.videoEditor = new VideoEditor(state); this.soundFxEditor = new SoundFxEditor(state); this.instrumentEditor = new InstrumentEditor(state); this.patternEditor = new PatternEditor(state, sounds); @@ -449,9 +457,9 @@ export class Editor implements PlayHost { this.workspace = new HStack(); this.leftColumn = new VStack(); - this.assets = new AssetPanel(state, textures, fonts, sounds); + this.assets = new AssetPanel(state, textures, fonts, sounds, videos); this.leftColumn.add(this.assets, this.assetsShare); - this.cleanups.push(installFileImport(app.canvas, { state, textures, fonts, sounds })); + this.cleanups.push(installFileImport(app.canvas, { state, textures, fonts, sounds, videos })); this.leftColumn.add(this.columnSplitter); this.leftColumn.add(this.hierarchy, 1 - this.assetsShare); // A share rather than a height, so a taller or shorter window keeps the split the user chose. @@ -493,6 +501,7 @@ export class Editor implements PlayHost { shader: this.shaderEditor, particles: this.particlesEditor, sound: this.soundEditor, + video: this.videoEditor, soundfx: this.soundFxEditor, instrument: this.instrumentEditor, pattern: this.patternEditor, @@ -1340,6 +1349,7 @@ export class Editor implements PlayHost { if (this.shows('song')) this.songEditor.tick(); else this.songEditor.hideMeters(); if (this.shows('sound')) this.soundEditor.tick(); + if (this.shows('video')) this.videoEditor.tick(); // The tabs of an EQ or a Compressor and their nodes (#380, #381) listen to the same taps: what // is not seen lets go of them first, so it cannot take them from what is. const faces = [ @@ -1392,6 +1402,7 @@ export class Editor implements PlayHost { (this.shows('pattern') && this.patternEditor.isPlaying) || (this.shows('song') && this.songEditor.isPlaying) || (this.shows('sound') && this.soundEditor.isPlaying) || + (this.shows('video') && this.videoEditor.isPlaying) || (this.shows('eq') && this.eqEditor.hasSpectrum) || (this.shows('filter') && this.filterEditor.hasSpectrum) || (this.shows('compressor') && this.compressorEditor.moving) || diff --git a/src/editor/PlaySession.ts b/src/editor/PlaySession.ts index 0ff2684d..e765a235 100644 --- a/src/editor/PlaySession.ts +++ b/src/editor/PlaySession.ts @@ -16,6 +16,7 @@ import { Input } from '../runtime/systems/Input'; import type { World } from '../runtime/World'; import { ui } from '../ui'; import { soundLibrary } from '../viewport/SoundLibrary'; +import { videoLibrary } from '../viewport/VideoLibrary'; import { FrameMeter, type FrameTimes } from './frameMeter'; import { saveArea } from './saves'; import { PanelClock, panelPolicy, type PanelUpdate } from './playThrottle'; @@ -108,6 +109,7 @@ export class PlaySession { // A held note is part of the picture a pause freezes: the sounds stop where they are and carry // on mid-note, rather than playing to their end over a still image. soundLibrary.setSuspended(this.state.paused); + videoLibrary.setPaused(this.state.paused); ui().focus.blur(); this.state.setNotice(this.playNotice(), 'quiet'); } @@ -162,6 +164,7 @@ export class PlaySession { saves: saveArea(runtime.id, 'play'), }; this.run = GameRun.start(runtime, scene.id, this.input, services); + videoLibrary.startRun(); this.updates = { structure: 0, values: 0 }; this.meter.reset(); this.timeSystems(this.run); @@ -212,6 +215,7 @@ export class PlaySession { // And its sounds end with it, looping music included – nothing may outlive the world. soundLibrary.setSuspended(false); soundLibrary.stopAll(); + videoLibrary.endRun(); const world = this.world; this.run = null; this.input.detach(); diff --git a/src/editor/actions.ts b/src/editor/actions.ts index 9be3e9cb..873b8492 100644 --- a/src/editor/actions.ts +++ b/src/editor/actions.ts @@ -32,6 +32,7 @@ import { ownerSong } from '../core/music/embedded'; import type { SceneObject } from '../core/model/SceneObject'; import { pixelArtRefIds } from '../core/model/pixelart'; import { textureRefIds } from '../core/model/texture'; +import { videoRefIds } from '../core/model/video'; import { animationsUsing, cellsPainted, @@ -252,10 +253,12 @@ export function confirmDeleteAsset(state: EditorState, assetId: string): void { ? textureRefIds(asset) : asset.type === 'pixelart' ? pixelArtRefIds(asset) - : [assetId]; + : asset.type === 'video' + ? videoRefIds(asset) + : [assetId]; const references = findPropertyUsages(state.project, ids).length; // An asset that draws its own picture (#185) is worn the same way, only without frames. - const picture = asset.type === 'texture' || generatesPictures(asset.type); + const picture = asset.type === 'texture' || asset.type === 'video' || generatesPictures(asset.type); const animations = picture ? animationsUsing(state.project, ids).length : 0; const tiles = picture ? tilesUsing(state.project, ids).length : 0; const eased = asset.type === 'curve' ? curveUsages(state.project, assetId) : { keys: 0, nodes: 0 }; diff --git a/src/editor/assetImport.ts b/src/editor/assetImport.ts index f019227d..16b215b1 100644 --- a/src/editor/assetImport.ts +++ b/src/editor/assetImport.ts @@ -147,7 +147,7 @@ export function assetNameForFile(fileName: string, fallback: string): string { } /** Opens the browser's file dialog; resolves empty when the dialog was cancelled. */ -function pickFiles(accept: string): Promise { +export function pickFiles(accept: string): Promise { return new Promise((resolve) => { const input = document.createElement('input'); input.type = 'file'; diff --git a/src/editor/assetTabs.ts b/src/editor/assetTabs.ts index b74f13da..8ac38613 100644 --- a/src/editor/assetTabs.ts +++ b/src/editor/assetTabs.ts @@ -10,6 +10,7 @@ export const TAB_OF_ASSET: Partial> = { shaderfunction: 'shader', particles: 'particles', sound: 'sound', + video: 'video', soundfx: 'soundfx', instrument: 'instrument', pattern: 'pattern', diff --git a/src/editor/debug/videoHooks.ts b/src/editor/debug/videoHooks.ts new file mode 100644 index 00000000..42f49765 --- /dev/null +++ b/src/editor/debug/videoHooks.ts @@ -0,0 +1,40 @@ +import type { VideoClip } from '../../core/model/Asset'; +import { findAssetOf } from '../../core/model/Project'; +import { clipsOf } from '../../core/model/sound'; +import { videoLibrary } from '../../viewport/VideoLibrary'; +import { currentVideoPicker } from '../video/VideoPicker'; +import { centreOf, type HookContext } from './hookHelpers'; + +/** The Video tab, its picker and the players behind the sprites (#402). */ +export function videoHooks({ state, editor }: HookContext) { + const { videoEditor } = editor; + return { + videoTab: { + current: () => videoEditor.currentAssetId, + info: () => videoEditor.shownInfo, + rect: () => videoEditor.canvasRect(), + span: () => videoEditor.canvas.shownSpan(), + playhead: () => videoEditor.canvas.playhead, + playing: () => videoEditor.isPlaying, + playButton: () => ({ ...centreOf(videoEditor.playButton), text: videoEditor.playButton.text }), + preview: () => videoEditor.canvas.previewInfo(), + clips: () => { + const asset = findAssetOf(state.project, videoEditor.currentAssetId, 'video'); + return asset + ? clipsOf(asset).map((c) => ({ id: c.id, name: c.name, props: { ...c.props } })) + : []; + }, + pickedClip: () => videoEditor.canvas.pickedClip()?.id ?? null, + /** The middle of the clip strip, and of the track under it, at a second of the video. */ + stripAt: (seconds: number) => videoEditor.canvas.stripAt(seconds), + trackAt: (seconds: number) => videoEditor.canvas.trackAt(seconds), + }, + videoPicker: () => { + const picker = currentVideoPicker(); + return picker ? { ...picker.debugInfo(), waiting: picker.waitingFor } : null; + }, + videoPickerRow: (file: string) => currentVideoPicker()?.rowGlobal(file) ?? null, + /** What each player behind a sprite is doing: its time, whether it runs and is heard. */ + videos: () => videoLibrary.debugInfo(), + }; +} diff --git a/src/editor/debugHook.ts b/src/editor/debugHook.ts index b58b657f..917e9c05 100644 --- a/src/editor/debugHook.ts +++ b/src/editor/debugHook.ts @@ -11,6 +11,7 @@ import { compressorMakeup } from '../core/soundfx/dynamics'; import { bakeHooks } from './debug/bakeHooks'; import { curveHooks } from './debug/curveHooks'; import { soundHooks } from './debug/soundHooks'; +import { videoHooks } from './debug/videoHooks'; import { patternHooks } from './debug/patternHooks'; import { routingHooks } from './debug/routingHooks'; import { songHooks } from './debug/songHooks'; @@ -49,6 +50,7 @@ export function installDebugHook(app: Application, state: EditorState, editor: E ...songHooks(context), ...routingHooks(context), ...soundHooks(context), + ...videoHooks(context), ...bakeHooks(context), ...renderHooks(context), /** diff --git a/src/editor/fileImport.ts b/src/editor/fileImport.ts index 4e1a77fc..ec959eec 100644 --- a/src/editor/fileImport.ts +++ b/src/editor/fileImport.ts @@ -1,14 +1,16 @@ import { type FontImportDeps, importFonts, isFontFileName } from './fontImport'; import { importSounds, isSoundFileName, type SoundImportDeps } from './soundImport'; import { importImages, type TextureImportDeps } from './textureImport'; +import { openVideoPicker } from './video/VideoPicker'; +import { isVideoFileName, uploadVideos, type VideoImportDeps } from './videoImport'; -export type FileImportDeps = TextureImportDeps & FontImportDeps & SoundImportDeps; +export type FileImportDeps = TextureImportDeps & FontImportDeps & SoundImportDeps & VideoImportDeps; /** * Wires drop-onto-`target` and paste, and returns the remover for both. Paste is bound to the * window because the canvas cannot hold focus, and skipped while a DOM text field has it. * - * Pictures, typefaces and sounds arrive the same way, so one pair of handlers sorts them apart. + * Pictures, typefaces, sounds and videos arrive the same way, so one pair of handlers sorts them apart. */ export function installFileImport(target: HTMLElement, deps: FileImportDeps): () => void { const onDragOver = (event: DragEvent) => { @@ -22,7 +24,7 @@ export function installFileImport(target: HTMLElement, deps: FileImportDeps): () if (files.none) { if (!event.dataTransfer?.files.length) return; event.preventDefault(); - deps.state.setNotice('Only image, font and sound files can be dropped into the editor.'); + deps.state.setNotice('Only image, font, sound and video files can be dropped into the editor.'); return; } event.preventDefault(); @@ -50,6 +52,7 @@ interface SortedFiles { images: File[]; fonts: File[]; sounds: File[]; + videos: File[]; none: boolean; } @@ -58,7 +61,14 @@ function sortFiles(data: DataTransfer | null): SortedFiles { const images = all.filter((file) => file.type.startsWith('image/')); const fonts = all.filter((file) => isFontFileName(file.name)); const sounds = all.filter((file) => isSoundFileName(file.name)); - return { images, fonts, sounds, none: images.length + fonts.length + sounds.length === 0 }; + const videos = all.filter((file) => isVideoFileName(file.name)); + return { + images, + fonts, + sounds, + videos, + none: images.length + fonts.length + sounds.length + videos.length === 0, + }; } /** One kind after the other, so a mixed drop ends on one notice instead of three racing ones. */ @@ -67,6 +77,9 @@ function run(files: SortedFiles, deps: FileImportDeps): void { if (files.images.length) await importImages(files.images, deps); if (files.fonts.length) await importFonts(files.fonts, deps); if (files.sounds.length) await importSounds(files.sounds, deps); + // A video is converted first (#402): the picker opens to show how far, and adopts it when done. + if (files.videos.length) + await uploadVideos(files.videos, deps, (pending) => openVideoPicker(deps, pending)); }; void work().catch((error: unknown) => deps.state.setNotice(String(error), 'error')); } diff --git a/src/editor/inspector/SpritePreview.ts b/src/editor/inspector/SpritePreview.ts index 8deedcce..2f6ff2fd 100644 --- a/src/editor/inspector/SpritePreview.ts +++ b/src/editor/inspector/SpritePreview.ts @@ -1,4 +1,5 @@ import { Graphics } from 'pixi.js'; +import { isVideoKey } from '../../core/model/video'; import { SHEET_EDGE } from '../../core/shader/nodes'; import { crisp, hairline, type Size, Theme, ui, Widget } from '../../ui'; import { cancelFrameWork, onFrame } from '../../ui/render'; @@ -91,7 +92,9 @@ export class SpritePreview extends Widget { shaders, ); this.drawn = { x, y, w, h }; - this.animates = Boolean(worn?.time) && this.onScreen(); + // A video (#402) moves by itself, and its player is let go once nobody asks for its picture. + const video = picture.texture !== null && isVideoKey(picture.texture.file); + this.animates = (Boolean(worn?.time) || video) && this.onScreen(); if (this.animates) onFrame(this.frameKey, () => this.invalidate()); } diff --git a/src/editor/inspector/clipSection.ts b/src/editor/inspector/clipSection.ts index 8e222f55..2eaab7b2 100644 --- a/src/editor/inspector/clipSection.ts +++ b/src/editor/inspector/clipSection.ts @@ -1,8 +1,10 @@ import { RemoveClipCommand, RenameClipCommand, SetClipPropsCommand } from '../../core/commands/clipCommands'; -import type { SoundAsset, SoundClip } from '../../core/model/Asset'; -import { findAssetOf, type Project } from '../../core/model/Project'; -import { clipsOf } from '../../core/model/sound'; +import type { SoundClip } from '../../core/model/Asset'; +import { findAsset, type Project } from '../../core/model/Project'; +import { type ClipOwner, clipsOf, hasClips } from '../../core/model/sound'; +import type { PropertyDef } from '../../core/properties/PropertySchema'; import { CLIP_ROWS } from '../../core/properties/sounds'; +import { VIDEO_CLIP_ROWS } from '../../core/properties/videos'; import { Button } from '../../ui'; import { createFieldRow } from './fieldFactory'; import { addGap, addNameField, addNote, addTitle, type InspectorHost } from './host'; @@ -13,13 +15,17 @@ export function clipSignature(project: Project, assetId: string, clipId: string) return currentClip(project, assetId, clipId) ? 'clip' : 'missing'; } -/** Name, the stretch of the file it is and how it is played, and the way to delete it (#357). */ +/** + * Name, the stretch of the file it is and how it is played, and the way to delete it (#357). A clip + * of a video (#402) is only its stretch. + */ export function buildClipSection(host: InspectorHost, assetId: string, clipId: string): void { - const asset = currentSound(host.state.project, assetId); + const asset = currentOwner(host.state.project, assetId); const clip = asset && currentClip(host.state.project, assetId, clipId); if (!asset || !clip) return; + const video = asset.type === 'video'; - addTitle(host, 'SOUND CLIP'); + addTitle(host, video ? 'VIDEO CLIP' : 'SOUND CLIP'); addNameField(host, clip.name, { onChange: (name) => { // The command rejects a blank name but cannot put the old text back into the field. @@ -27,9 +33,14 @@ export function buildClipSection(host: InspectorHost, assetId: string, clipId: s else host.nameField?.setValue(clip.name); }, }); - addNote(host, `Clip of the sound "${asset.name}". Its volume, loop, bus and sound FX are the sound's.`); + addNote( + host, + video + ? `Clip of the video "${asset.name}". Its volume and loop are the video's.` + : `Clip of the sound "${asset.name}". Its volume, loop, bus and sound FX are the sound's.`, + ); - for (const def of CLIP_ROWS) { + for (const def of rowsOf(asset)) { const handlers = previewingHandlers(host, { target: () => currentClip(host.state.project, assetId, clipId), read: (target) => target.props[def.key], @@ -57,16 +68,23 @@ export function refreshClipSection(host: InspectorHost, assetId: string, clipId: const clip = currentClip(host.state.project, assetId, clipId); if (!clip) return false; host.nameField?.setValue(clip.name); - for (const def of CLIP_ROWS) host.fields.get(def.key)?.setValue(clip.props[def.key] ?? def.default); + const asset = currentOwner(host.state.project, assetId); + for (const def of asset ? rowsOf(asset) : []) + host.fields.get(def.key)?.setValue(clip.props[def.key] ?? def.default); return true; } -/** Looked up inside the sound, so a selection left over from a deleted sound edits nothing. */ -function currentClip(project: Project, assetId: string, clipId: string): SoundClip | undefined { - const asset = currentSound(project, assetId); - return asset && clipsOf(asset).find((c) => c.id === clipId); +function rowsOf(asset: ClipOwner): readonly PropertyDef[] { + return asset.type === 'video' ? VIDEO_CLIP_ROWS : CLIP_ROWS; } -function currentSound(project: Project, assetId: string): SoundAsset | undefined { - return findAssetOf(project, assetId, 'sound'); +/** Looked up inside its asset, so a selection left over from a deleted one edits nothing. */ +function currentClip(project: Project, assetId: string, clipId: string): SoundClip | undefined { + const asset = currentOwner(project, assetId); + return asset && clipsOf(asset).find((c) => c.id === clipId); +} + +function currentOwner(project: Project, assetId: string): ClipOwner | undefined { + const asset = findAsset(project, assetId); + return asset && hasClips(asset) ? asset : undefined; } diff --git a/src/editor/inspector/fieldFactory.ts b/src/editor/inspector/fieldFactory.ts index 58e9b002..d1376296 100644 --- a/src/editor/inspector/fieldFactory.ts +++ b/src/editor/inspector/fieldFactory.ts @@ -34,7 +34,12 @@ import { type StructFieldDef, } from '../../core/properties/PropertySchema'; import { referencedAssetId } from '../../core/properties/references'; -import { NO_TEXTURE_LABEL, pictureChoices, textureRefLabel } from '../../core/properties/textureRef'; +import { + NO_TEXTURE_LABEL, + pictureChoices, + textureRefLabel, + videoChoices, +} from '../../core/properties/textureRef'; import { Button, CHEVRON_SLOT, @@ -418,10 +423,12 @@ const REFERENCES: Record = { texture: { none: NO_TEXTURE_LABEL, label: textureRefLabel, - choices: (project, def) => - pictureChoices(project).filter( + choices: (project, def) => [ + ...pictureChoices(project).filter( (choice) => !(def.kind === 'texture' && def.uploadedOnly) || choice.asset.type === 'texture', ), + ...(def.kind === 'texture' && def.moving ? videoChoices(project) : []), + ], }, statemachine: { none: NO_ASSET_LABEL, diff --git a/src/editor/panels/AssetPanel.ts b/src/editor/panels/AssetPanel.ts index 64bf7ca7..d4ae4fe9 100644 --- a/src/editor/panels/AssetPanel.ts +++ b/src/editor/panels/AssetPanel.ts @@ -24,6 +24,7 @@ import { schemaForAsset } from '../../core/properties/PropertySchema'; import { resolveAssetProps } from '../../core/properties/resolve'; import type { FontStore } from '../../core/store/FontStore'; import type { SoundStore } from '../../core/store/SoundStore'; +import type { VideoStore } from '../../core/store/VideoStore'; import type { TextureStore } from '../../core/store/TextureStore'; import { Button, @@ -46,6 +47,7 @@ import { generalSettings, onGeneralSettings } from '../generalSettings'; import type { EditorState } from '../state'; import { openFontPicker } from '../font/FontPicker'; import { openSoundPicker } from '../sound/SoundPicker'; +import { openVideoPicker } from '../video/VideoPicker'; import { openImagePicker } from '../texture/ImagePicker'; /** Above every menu of new assets, so a click on a type reads as making one (#361). */ @@ -84,6 +86,8 @@ export class AssetPanel extends Panel { private readonly fonts: FontStore | null = null, /** And for sounds and the Sound entry. */ private readonly sounds: SoundStore | null = null, + /** And for videos and the Video entry (#402). */ + private readonly videos: VideoStore | null = null, ) { super({ title: 'Assets', padding: 0, gap: 0 }); this.minWidth = 160; @@ -338,7 +342,7 @@ export class AssetPanel extends Panel { } /** - * A texture is a picture, a font and a sound are files, so their menu entries offer what the + * A texture is a picture, a font, a sound and a video are files, so their menu entries offer what the * server already holds instead of creating an empty asset. Uploading a new one is a button * inside that dialog. */ @@ -346,6 +350,7 @@ export class AssetPanel extends Panel { if (type === 'texture') this.importTextures(); else if (type === 'font') this.importFonts(); else if (type === 'sound') this.importSounds(); + else if (type === 'video') this.importVideos(); else if (type === 'pixelart') this.createPixelArt(); else this.createAsset(type); } @@ -393,6 +398,14 @@ export class AssetPanel extends Panel { openSoundPicker({ state: this.state, sounds: this.sounds }); } + private importVideos(): void { + if (!this.videos) { + this.state.setNotice('Video upload is not available.'); + return; + } + openVideoPicker({ state: this.state, videos: this.videos }); + } + /** Double-click / Enter opens the asset's tab; a scene also becomes the active one. Placing is a drag. */ private activate(id: string): void { activateAsset(this.state, id); @@ -471,6 +484,12 @@ export class AssetPanel extends Panel { if (asset.type === 'pixelart' && asset.frames.length > 1) items.push({ label: 'Create animation from frames', onSelect: () => this.createAnimation(asset) }); } + // A video is worn like a texture (#402), at its own size. + if (asset.type === 'video') + items.push({ + label: 'Create sprite from video', + onSelect: () => this.createAsset('sprite', asset.name, spriteFromTexture(this.state.project, id)), + }); items.push({ label: 'Duplicate', onSelect: () => this.duplicateAsset(id) }); items.push({ label: 'Delete…', danger: true, onSelect: () => confirmDeleteAsset(this.state, id) }); const menu = new Menu(items); diff --git a/src/editor/panels/assetGroups.ts b/src/editor/panels/assetGroups.ts index 6c175fb5..b3044f00 100644 --- a/src/editor/panels/assetGroups.ts +++ b/src/editor/panels/assetGroups.ts @@ -28,6 +28,7 @@ export const ASSET_GROUPS: Record = { animation, @@ -72,6 +73,7 @@ export const ASSET_ICON_SVG: Record = { tilemap, variables, vectorart, + video, }; /** A Variables tab showing one list (#252): a list value is no asset. */ diff --git a/src/editor/panels/icons/video.svg b/src/editor/panels/icons/video.svg new file mode 100644 index 00000000..a97764cb --- /dev/null +++ b/src/editor/panels/icons/video.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/editor/shader/ShaderEditor.ts b/src/editor/shader/ShaderEditor.ts index 00584264..f2b84671 100644 --- a/src/editor/shader/ShaderEditor.ts +++ b/src/editor/shader/ShaderEditor.ts @@ -123,6 +123,7 @@ export class ShaderEditor extends AssetTabEditor { /** Called every frame while the tab is showing. */ tick(): void { + if (this.preview.wearsVideo) this.preview.invalidate(); if (!this.animates) return; this.preview.invalidate(); this.syncControls(); diff --git a/src/editor/shader/ShaderPreview.ts b/src/editor/shader/ShaderPreview.ts index f788bd96..d04ec9be 100644 --- a/src/editor/shader/ShaderPreview.ts +++ b/src/editor/shader/ShaderPreview.ts @@ -1,5 +1,6 @@ import { Container, Graphics, Texture } from 'pixi.js'; import type { ShaderAsset } from '../../core/model/Asset'; +import { isVideoKey } from '../../core/model/video'; import type { PropertyBag } from '../../core/model/Properties'; import { SHADER_KEY } from '../../core/properties/shaders'; import { type EdgeMode, SHEET_EDGE } from '../../core/shader/nodes'; @@ -60,6 +61,12 @@ export class ShaderPreview extends Widget { return this.held ?? Math.max(0, shaderLibrary.now() - this.clockOrigin); } + /** Whether the picture is a video (#402): it moves by itself and must be asked for every frame. */ + get wearsVideo(): boolean { + const ref = this.picture?.texture; + return Boolean(ref && isVideoKey(ref.file)); + } + get holding(): boolean { return this.held !== null; } diff --git a/src/editor/state.ts b/src/editor/state.ts index 7105d8c5..bbc02c82 100644 --- a/src/editor/state.ts +++ b/src/editor/state.ts @@ -21,7 +21,7 @@ import { type Project, startScene, } from '../core/model/Project'; -import { clipsOf } from '../core/model/sound'; +import { clipsOf, hasClips } from '../core/model/sound'; import type { PropertyValue } from '../core/model/Properties'; import { deserializeProject, serializeProject } from '../core/store/serialize'; import { requestRender } from '../ui'; @@ -57,7 +57,7 @@ export type Selection = } /** A frame rectangle inside a texture asset (the Texture tab). */ | { kind: 'frame'; assetId: string; frameId: string } - /** A clip of a sound asset (the Sound tab, #357). */ + /** A clip of a sound asset (the Sound tab, #357) or of a video (the Video tab, #402). */ | { kind: 'clip'; assetId: string; clipId: string } /** * One or more shapes of a vector art asset (the Vector art tab, #129). `shapeId` is the primary @@ -797,10 +797,11 @@ export class EditorState { this.select({ kind: 'asset', assetId: s.assetId }); } if (s.kind === 'clip') { - const asset = findAssetOf(this.project, s.assetId, 'sound'); - // As with a frame: the sound stays open when its clip vanished. + const found = findAsset(this.project, s.assetId); + const asset = found && hasClips(found) ? found : undefined; + // As with a frame: the sound or video stays open when its clip vanished. if (!asset) this.select(null); - else if (!clipsOf(asset).some((c) => c.id === s.clipId)) + else if (!clipsOf<{ id: string }>(asset).some((c) => c.id === s.clipId)) this.select({ kind: 'asset', assetId: s.assetId }); } if (s.kind === 'shape') { @@ -893,7 +894,14 @@ export class EditorState { return findAssetOf(this.project, s.assetId, 'sound')?.id ?? null; } - /** The clip of a sound the selection is about (#357); empty for the sound itself. */ + /** The video the selection refers to (#402): the asset, or one of its clips. */ + get selectedVideoId(): string | null { + const s = this.selection; + if (s?.kind !== 'asset' && s?.kind !== 'clip') return null; + return findAssetOf(this.project, s.assetId, 'video')?.id ?? null; + } + + /** The clip of a sound or video the selection is about (#357); empty for the asset itself. */ get selectedClipId(): string { return this.selection?.kind === 'clip' ? this.selection.clipId : ''; } diff --git a/src/editor/video/VideoCanvas.ts b/src/editor/video/VideoCanvas.ts new file mode 100644 index 00000000..91333b83 --- /dev/null +++ b/src/editor/video/VideoCanvas.ts @@ -0,0 +1,633 @@ +import { + Container, + type FederatedPointerEvent, + type FederatedWheelEvent, + Graphics, + Sprite, + type Text, + Texture, +} from 'pixi.js'; +import { AddClipCommand, SetClipPropsCommand } from '../../core/commands/clipCommands'; +import type { VideoAsset, VideoClip } from '../../core/model/Asset'; +import type { PropertyBag, Vec2 } from '../../core/model/Properties'; +import { findAssetOf } from '../../core/model/Project'; +import { clipsOf } from '../../core/model/sound'; +import { + videoLoopsOf, + videoSpanOf, + videoVolumeOf, + VIDEO_END_KEY, + VIDEO_START_KEY, +} from '../../core/properties/videos'; +import { crispText, type Size, textStyle, Theme, ui, Widget, zoomFactor } from '../../ui'; +import { + clampWave, + type ClipBand, + clipAt, + type CutSpan, + draggedSpan, + edgeAt, + fitWave, + rulerLabel, + rulerStep, + secondsToX, + spanOf, + storedCut, + type WaveView, + xToSeconds, + zoomWave, +} from '../sound/waveView'; +import { openVideo } from '../../viewport/VideoLibrary'; +import type { EditorState } from '../state'; +import { containRect } from '../texture/imageGrid'; + +const RULER_HEIGHT = 24; +/** The strip the clips stand in as bands, as in the Sound tab. */ +const STRIP_HEIGHT = 20; +/** The track under it, where the picked clip's edges are dragged. */ +const TRACK_HEIGHT = 32; +const TIMELINE_HEIGHT = RULER_HEIGHT + STRIP_HEIGHT + TRACK_HEIGHT; +const FIT_MARGIN = 16; +const LABEL_SPACING = 70; +/** A clip is a frame long at least; at 60 fps that is this. */ +const LEAST_SPAN = 1 / 60; +const DRAG_START = 3; +/** Clip edges are stored to the millisecond. */ +const STORED_RATE = 1000; +const PREVIEW_PAD = 8; +const LIT_LABEL = textStyle('small').clone(); +LIT_LABEL.fill = 0x30343c; + +type Gesture = + | { kind: 'pan'; pointerX: number; panX: number } + | { kind: 'seek' } + | { + kind: 'span'; + clipId: string | null; + edge: 'start' | 'end' | null; + from: number; + pointerX: number; + moved: boolean; + span: CutSpan; + picks?: string | null; + }; + +/** The tab's own player: the video shown, with its sound, apart from the ones sprites wear. */ +interface Preview { + file: string; + element: HTMLVideoElement; + /** Made once the video has shown its first frame, see `openVideo`. */ + texture: Texture | null; +} + +/** + * The Video tab's canvas (#402): the video on top, a ruler in seconds under it, the clips as bands + * in a strip, and a track where the picked clip stands between two handles. A click in the strip + * picks a clip, a drag there or in the track spans a new one; dragging a handle moves that edge. + * One drag is one undo step. A click in the ruler shows that moment; the wheel zooms the time, the + * middle button pans, as in the Sound tab. + */ +/** The colour in the middle of the frame a video holds; transparent where it has none to draw. */ +export function middlePixel(element: HTMLVideoElement): number[] | null { + if (element.readyState < 2) return null; + const context = new OffscreenCanvas(8, 8).getContext('2d', { willReadFrequently: true }); + if (!context) return null; + context.drawImage(element, 0, 0, 8, 8); + return [...context.getImageData(4, 4, 1, 1).data]; +} + +export class VideoCanvas extends Widget { + private readonly bg = new Graphics(); + private readonly picture = new Sprite(); + private readonly overlay = new Graphics(); + private readonly labels = new Container(); + private readonly texts: Text[] = []; + private readonly stripLabels = new Container(); + private readonly stripTexts: Text[] = []; + private readonly hint = crispText({ text: '', style: textStyle('muted') }); + private view: WaveView = { panX: FIT_MARGIN, pxPerSecond: 100 }; + private needsFit = true; + private assetId = ''; + private gesture: Gesture | null = null; + private preview: Preview | null = null; + /** The span the player is held to while it plays: the picked clip, else the whole video. */ + private playing: CutSpan | null = null; + onTogglePlay: () => void = () => {}; + /** The player stopped by itself (the end of a clip that does not repeat). */ + onStopped: () => void = () => {}; + + constructor(private readonly state: EditorState) { + super(); + this.focusable = true; + this.labels.eventMode = 'none'; + this.stripLabels.eventMode = 'none'; + this.hint.eventMode = 'none'; + this.picture.eventMode = 'none'; + this.addChild(this.bg, this.picture, this.overlay, this.labels, this.stripLabels, this.hint); + this.on('pointerdown', (e: FederatedPointerEvent) => this.onPointerDown(e)); + this.on('globalpointermove', (e: FederatedPointerEvent) => this.onPointerMove(e)); + this.onPointerRelease(() => this.onPointerUp()); + this.on('wheel', (e: FederatedWheelEvent) => this.onWheel(e)); + state.on('project', () => this.invalidate()); + state.on('selection', () => { + if (!this.isPlaying) this.seekTo(this.shownSpan()?.start ?? 0); + this.invalidate(); + }); + } + + override measure(): Size { + return { w: Math.max(this.minWidth, 360), h: Math.max(this.minHeight, TIMELINE_HEIGHT + 120) }; + } + + get currentAssetId(): string { + return this.assetId; + } + + show(assetId: string): void { + if (this.assetId === assetId) return; + this.stop(); + this.gesture = null; + this.assetId = assetId; + this.needsFit = true; + this.redraw(); + } + + saveView(): WaveView { + return { ...this.view }; + } + + restoreView(view: unknown): void { + const held = view as Partial; + if (typeof held.panX !== 'number' || typeof held.pxPerSecond !== 'number') return; + this.view = { panX: held.panX, pxPerSecond: held.pxPerSecond }; + this.needsFit = false; + this.redraw(); + } + + fit(): void { + const asset = this.asset(); + if (!asset || asset.duration <= 0 || this.w <= 0) { + this.needsFit = true; + return; + } + this.view = fitWave(asset.duration, this.w, FIT_MARGIN); + this.needsFit = false; + this.redraw(); + } + + get isPlaying(): boolean { + return this.playing !== null; + } + + /** Plays the picked clip, else the whole video, from its start, with the video's volume and loop. */ + play(): void { + const asset = this.asset(); + const span = this.shownSpan(); + const element = asset && this.player(asset); + if (!asset || !span || !element) return; + this.playing = span; + element.muted = false; + element.volume = videoVolumeOf(asset.props); + element.currentTime = span.start; + void element.play().catch(() => {}); + this.invalidate(); + } + + stop(): void { + this.playing = null; + this.preview?.element.pause(); + this.invalidate(); + } + + /** Where the tab's player stands, in seconds of the video; null while nothing is shown. */ + get playhead(): number | null { + return this.preview && this.preview.file === this.asset()?.file ? this.preview.element.currentTime : null; + } + + /** Called every frame while the tab is on screen: the playhead follows the player, a clip loops. */ + tick(): void { + const element = this.preview?.element; + const span = this.playing; + const asset = this.asset(); + if (!element || !span || !asset) return; + if (element.currentTime >= span.end - LEAST_SPAN / 2 || element.ended) { + if (videoLoopsOf(asset.props)) { + element.currentTime = span.start; + if (element.paused) void element.play().catch(() => {}); + } else { + this.stop(); + this.onStopped(); + } + } + this.redraw(); + } + + /** The picked clip of the video shown, null for none. */ + pickedClip(): VideoClip | null { + const asset = this.asset(); + const id = this.state.selectedClipId; + return (asset && id && clipsOf(asset).find((c) => c.id === id)) || null; + } + + /** What the tab plays: a drag's span while one runs, else the picked clip, else the whole video. */ + shownSpan(): CutSpan | null { + const asset = this.asset(); + if (!asset || asset.duration <= 0) return null; + if (this.gesture?.kind === 'span' && this.gesture.moved) return this.gesture.span; + const clip = this.pickedClip(); + return clip ? this.clipSpan(clip, asset) : { start: 0, end: asset.duration }; + } + + /** The middle of the strip at a second of the video, on screen (test hook). */ + stripAt(seconds: number): Vec2 { + const origin = this.getGlobalPosition(); + return { + x: origin.x + secondsToX(this.view, seconds), + y: origin.y + this.timelineTop + RULER_HEIGHT + STRIP_HEIGHT / 2, + }; + } + + /** The middle of the track at a second of the video, on screen (test hook). */ + trackAt(seconds: number): Vec2 { + const origin = this.getGlobalPosition(); + return { x: origin.x + secondsToX(this.view, seconds), y: origin.y + this.h - TRACK_HEIGHT / 2 }; + } + + /** + * Where the picture is drawn, whether the player has one, and the colour in the middle of the + * frame the player holds, read from the video itself (test hook). + */ + previewInfo(): { + shown: boolean; + x: number; + y: number; + w: number; + h: number; + time: number; + paused: boolean; + pixel: number[] | null; + } { + const p = this.picture.getGlobalPosition(); + return { + pixel: this.preview ? middlePixel(this.preview.element) : null, + shown: this.picture.visible, + x: p.x, + y: p.y, + w: this.picture.width, + h: this.picture.height, + time: this.preview?.element.currentTime ?? 0, + paused: this.preview?.element.paused ?? true, + }; + } + + override destroy(): void { + this.dropPreview(); + super.destroy({ children: true }); + } + + private get timelineTop(): number { + return Math.max(0, this.h - TIMELINE_HEIGHT); + } + + private asset(): VideoAsset | null { + return (this.assetId && findAssetOf(this.state.project, this.assetId, 'video')) || null; + } + + private clipSpan(clip: VideoClip, asset: VideoAsset): CutSpan { + const { start, end } = videoSpanOf(clip.props); + return spanOf(start, end, asset.duration); + } + + private bands(asset: VideoAsset): ClipBand[] { + const g = this.gesture?.kind === 'span' && this.gesture.moved ? this.gesture : null; + const bands = clipsOf(asset).map((clip) => ({ + id: clip.id, + span: g?.clipId === clip.id ? g.span : this.clipSpan(clip, asset), + })); + return g && g.clipId === null ? [...bands, { id: '', span: g.span }] : bands; + } + + /** The tab's player for the video's file, made anew when the file is another one. */ + private player(asset: VideoAsset): HTMLVideoElement | null { + if (!asset.file) return null; + if (this.preview?.file === asset.file) return this.preview.element; + this.dropPreview(); + const held = this.shownSpan()?.start ?? 0; + const preview: Preview = { + file: asset.file, + texture: null, + element: openVideo(`/game-maker/data/videos/${asset.file}`, held, (texture) => { + if (this.preview !== preview) return texture.destroy(true); + preview.texture = texture; + texture.source.on('update', () => this.invalidate()); + // It played for its first frame; until Play it stands at the moment the tab shows. + if (!this.playing) { + preview.element.pause(); + preview.element.currentTime = held; + } + this.invalidate(); + }), + }; + preview.element.addEventListener('seeked', () => this.invalidate()); + this.preview = preview; + return preview.element; + } + + private dropPreview(): void { + const preview = this.preview; + if (!preview) return; + this.preview = null; + this.playing = null; + this.picture.texture = Texture.EMPTY; + preview.element.pause(); + preview.element.removeAttribute('src'); + preview.element.load(); + preview.texture?.destroy(true); + } + + private seekTo(seconds: number): void { + const asset = this.asset(); + const element = asset && this.player(asset); + if (element) element.currentTime = Math.max(0, Math.min(asset.duration, seconds)); + } + + override onKeyDown(event: KeyboardEvent): boolean { + if (event.code !== 'Space' || event.ctrlKey || event.metaKey || !this.asset()) return false; + this.onTogglePlay(); + return true; + } + + private onPointerDown(e: FederatedPointerEvent): void { + const asset = this.asset(); + if (!asset || asset.duration <= 0) return; + const p = this.toLocal(e.global); + const y = p.y - this.timelineTop; + if (y < 0) return; + e.stopPropagation(); + ui().focus.focus(this); + if (e.button === 1) { + this.gesture = { kind: 'pan', pointerX: p.x, panX: this.view.panX }; + return; + } + if (e.button !== 0) return; + const from = Math.max(0, Math.min(asset.duration, xToSeconds(this.view, p.x))); + if (y < RULER_HEIGHT) { + this.gesture = { kind: 'seek' }; + this.stop(); + this.seekTo(from); + return; + } + const start = { + kind: 'span', + from, + pointerX: p.x, + moved: false, + span: { start: from, end: from }, + } as const; + if (y < RULER_HEIGHT + STRIP_HEIGHT) { + this.gesture = { ...start, clipId: null, edge: null, picks: clipAt(this.view, this.bands(asset), p.x) }; + return; + } + const clip = this.pickedClip(); + const span = clip && this.clipSpan(clip, asset); + const edge = span ? edgeAt(this.view, span, p.x) : null; + const beside = + span && !edge && (p.x < secondsToX(this.view, span.start) || p.x > secondsToX(this.view, span.end)); + this.gesture = { ...start, clipId: clip?.id ?? null, edge, picks: beside ? null : undefined }; + } + + private onPointerMove(e: FederatedPointerEvent): void { + const p = this.toLocal(e.global); + const g = this.gesture; + const asset = this.asset(); + if (!g || !asset) return; + if (g.kind === 'pan') { + this.view = clampWave({ ...this.view, panX: g.panX + p.x - g.pointerX }, asset.duration, this.w); + this.needsFit = false; + this.redraw(); + return; + } + if (g.kind === 'seek') { + this.seekTo(xToSeconds(this.view, p.x)); + return; + } + if (!g.moved && g.edge === null && Math.abs(p.x - g.pointerX) < DRAG_START) return; + g.moved = true; + const clip = this.pickedClip(); + const before = clip && clip.id === g.clipId ? this.clipSpan(clip, asset) : g.span; + g.span = draggedSpan(before, g.edge, g.from, xToSeconds(this.view, p.x), asset.duration, LEAST_SPAN); + this.seekTo(g.edge === 'end' ? g.span.end : g.span.start); + this.redraw(); + } + + private onPointerUp(): void { + const g = this.gesture; + this.gesture = null; + if (g?.kind !== 'span') return this.redraw(); + const asset = this.asset(); + if (!g.moved) { + if (asset && g.picks !== undefined) + this.state.select( + g.picks + ? { kind: 'clip', assetId: asset.id, clipId: g.picks } + : { kind: 'asset', assetId: asset.id }, + ); + return this.redraw(); + } + if (asset) this.commit(asset, g); + this.redraw(); + } + + /** A drag's span as the clip's start and end in one step, or as a new clip, which is then picked. */ + private commit(asset: VideoAsset, g: Extract): void { + const next = storedCut(g.span, asset.duration, STORED_RATE); + if (g.clipId === null) { + const add = new AddClipCommand(asset.id, { [VIDEO_START_KEY]: next.start, [VIDEO_END_KEY]: next.end }); + if (this.state.execute(add)) this.state.select({ kind: 'clip', assetId: asset.id, clipId: add.clipId }); + return; + } + const clip = clipsOf(asset).find((c) => c.id === g.clipId); + if (!clip) return; + const now = videoSpanOf(clip.props); + const values: PropertyBag = {}; + if (next.start !== now.start) values[VIDEO_START_KEY] = next.start; + if (next.end !== now.end) values[VIDEO_END_KEY] = next.end; + if (Object.keys(values).length > 0) + this.state.execute( + new SetClipPropsCommand(asset.id, clip.id, values, g.edge ? 'Move clip edge' : 'Span clip'), + ); + } + + private onWheel(e: FederatedWheelEvent): void { + const asset = this.asset(); + const p = this.toLocal(e.global); + if (!asset || asset.duration <= 0 || p.y < this.timelineTop) return; + this.view = clampWave(zoomWave(this.view, zoomFactor(e.deltaY), p.x), asset.duration, this.w); + this.needsFit = false; + this.redraw(); + e.preventDefault(); + } + + override layout(): void { + if (this.needsFit) this.fit(); + } + + override redraw(): void { + const asset = this.asset(); + if (this.needsFit && asset && asset.duration > 0 && this.w > 0) return this.fit(); + const c = Theme.colors; + const top = this.timelineTop; + this.bg.clear().rect(0, 0, this.w, this.h).fill(0x000000); + this.bg.rect(0, top, this.w, RULER_HEIGHT + STRIP_HEIGHT).fill(c.panelHeader); + this.bg.rect(0, top + RULER_HEIGHT - 1, this.w, 1).fill(c.border); + this.bg.rect(0, top + RULER_HEIGHT + STRIP_HEIGHT - 1, this.w, 1).fill(c.border); + this.bg.rect(0, top + RULER_HEIGHT + STRIP_HEIGHT, this.w, TRACK_HEIGHT).fill(c.viewport); + this.overlay.clear(); + this.hint.visible = !asset?.file; + this.hint.text = !asset ? 'No video selected – pick one in the Assets panel.' : 'Not uploaded.'; + this.hint.position.set(Theme.padding * 2, Theme.padding * 2); + if (!asset?.file) { + this.picture.visible = false; + this.syncLabels([]); + this.syncStrip([]); + return; + } + this.drawPicture(asset, top); + const left = Math.max(0, secondsToX(this.view, 0)); + const right = Math.min(this.w, secondsToX(this.view, asset.duration)); + this.drawRuler(left, right, top); + this.drawStrip(asset, top); + this.drawTrack(asset, left, right, top); + const head = this.playhead; + if (head !== null) { + const x = secondsToX(this.view, head); + if (x >= 0 && x <= this.w) this.overlay.rect(x - 0.5, top, 1, TIMELINE_HEIGHT).fill(c.hot); + } + } + + private drawPicture(asset: VideoAsset, top: number): void { + this.player(asset); + const texture = this.preview?.texture; + this.picture.visible = Boolean(texture); + if (!texture) return; + const room = { w: this.w - PREVIEW_PAD * 2, h: top - PREVIEW_PAD * 2 }; + const fitted = containRect(asset.width || texture.width, asset.height || texture.height, room.w, room.h); + this.picture.texture = texture; + this.picture.position.set(PREVIEW_PAD + fitted.x, PREVIEW_PAD + fitted.y); + this.picture.width = Math.max(1, fitted.w); + this.picture.height = Math.max(1, fitted.h); + } + + private drawRuler(left: number, right: number, top: number): void { + const step = rulerStep(this.view.pxPerSecond, LABEL_SPACING); + const placed: { text: string; x: number }[] = []; + const first = Math.ceil(Math.max(0, xToSeconds(this.view, left)) / step); + for (let k = first; ; k++) { + const seconds = k * step; + const x = secondsToX(this.view, seconds); + if (x > right + 0.5) break; + this.overlay.rect(x, top + RULER_HEIGHT - 8, 1, 7).fill({ color: Theme.colors.textMuted, alpha: 0.8 }); + placed.push({ text: rulerLabel(seconds, step), x: x + 3 }); + } + this.syncLabels(placed.map((label) => ({ ...label, y: top + 6 }))); + } + + private litClipId(): string { + const g = this.gesture?.kind === 'span' && this.gesture.moved ? this.gesture : null; + return (g ? g.clipId : this.pickedClip()?.id) ?? ''; + } + + private drawStrip(asset: VideoAsset, top: number): void { + const c = Theme.colors; + const lit = this.litClipId(); + const names = new Map(clipsOf(asset).map((clip) => [clip.id, clip.name])); + const placed: { text: string; x: number; room: number; lit: boolean }[] = []; + const y = top + RULER_HEIGHT; + for (const { id, span } of this.bands(asset)) { + const a = secondsToX(this.view, span.start); + const b = secondsToX(this.view, span.end); + if (b < 0 || a > this.w) continue; + const on = id === lit; + this.overlay + .rect(a, y + 2, Math.max(1, b - a), STRIP_HEIGHT - 4) + .fill({ color: c.accent, alpha: on ? 0.85 : 0.35 }) + .stroke({ width: 1, color: on ? c.accentText : c.accent, alpha: on ? 1 : 0.7, alignment: 1 }); + const name = names.get(id); + if (name) + placed.push({ + text: name, + x: Math.max(a, 0) + 4, + room: Math.min(b, this.w) - Math.max(a, 0) - 8, + lit: on, + }); + } + this.syncStrip(placed.map((label) => ({ ...label, y }))); + } + + /** The video's length lit, what lies outside the picked clip dark, and a handle at each of its edges. */ + private drawTrack(asset: VideoAsset, left: number, right: number, top: number): void { + const c = Theme.colors; + const y = top + RULER_HEIGHT + STRIP_HEIGHT; + const h = TRACK_HEIGHT; + this.overlay.rect(left, y + 4, Math.max(0, right - left), h - 8).fill({ color: c.panel, alpha: 0.8 }); + const clip = this.pickedClip(); + const span = + this.gesture?.kind === 'span' && this.gesture.moved + ? this.gesture.span + : clip && this.clipSpan(clip, asset); + if (!span) return; + const a = secondsToX(this.view, span.start); + const b = secondsToX(this.view, span.end); + this.overlay.rect(a, y + 4, Math.max(1, b - a), h - 8).fill({ color: c.accent, alpha: 0.35 }); + for (const [x, edge] of [ + [a, 'start'], + [b, 'end'], + ] as const) { + const held = this.gesture?.kind === 'span' && this.gesture.edge === edge; + this.overlay.rect(x - 1, y + 2, 2, h - 4).fill(held ? c.hot : c.accent); + const toward = edge === 'start' ? 1 : -1; + this.overlay + .moveTo(x, y + 2) + .lineTo(x + toward * 8, y + 2) + .lineTo(x, y + 12) + .closePath() + .fill(held ? c.hot : c.accent); + } + } + + private syncLabels(placed: readonly { text: string; x: number; y: number }[]): void { + placed.forEach((label, i) => { + let text = this.texts[i]; + if (!text) { + text = crispText({ style: textStyle('small') }); + this.texts.push(text); + this.labels.addChild(text); + } + text.visible = true; + if (text.text !== label.text) text.text = label.text; + text.position.set(Math.round(label.x), Math.round(label.y)); + }); + for (let i = placed.length; i < this.texts.length; i++) this.texts[i]!.visible = false; + } + + private syncStrip( + placed: readonly { text: string; x: number; y: number; room: number; lit: boolean }[], + ): void { + placed.forEach((label, i) => { + let text = this.stripTexts[i]; + if (!text) { + text = crispText({ style: textStyle('small') }); + this.stripTexts.push(text); + this.stripLabels.addChild(text); + } + const style = label.lit ? LIT_LABEL : textStyle('small'); + if (text.style !== style) text.style = style; + if (text.text !== label.text) text.text = label.text; + for (let cut = label.text.length; text.width > label.room && cut > 1; cut--) + text.text = `${label.text.slice(0, cut - 1)}…`; + text.visible = text.width <= label.room; + text.position.set(Math.round(label.x), Math.round(label.y + (STRIP_HEIGHT - text.height) / 2)); + }); + for (let i = placed.length; i < this.stripTexts.length; i++) this.stripTexts[i]!.visible = false; + } +} diff --git a/src/editor/video/VideoEditor.ts b/src/editor/video/VideoEditor.ts new file mode 100644 index 00000000..a66e7348 --- /dev/null +++ b/src/editor/video/VideoEditor.ts @@ -0,0 +1,84 @@ +import type { Asset, VideoAsset } from '../../core/model/Asset'; +import { Button } from '../../ui'; +import { AssetTabEditor } from '../AssetTabEditor'; +import { TOOLBAR_ICON_SVG } from '../panels/toolbarIcons'; +import type { EditorState } from '../state'; +import { VideoCanvas } from './VideoCanvas'; +import { formatDuration } from './videoInfo'; + +const INFO_WIDTH = 220; + +/** + * The Video tab (#402): the video with its clips, the stretches of it a sprite can wear instead of + * the whole video. The header plays the picked clip, else the whole video (Space does too). + */ +export class VideoEditor extends AssetTabEditor { + /** Public for the check scripts. */ + readonly playButton: Button; + + constructor(state: EditorState) { + super(state, { infoWidth: INFO_WIDTH }); + this.playButton = new Button('Play', { + icon: TOOLBAR_ICON_SVG.play, + minWidth: 72, + tooltip: 'Play the picked clip, else the whole video (Space)', + onClick: () => this.togglePlay(), + }); + const canvas = new VideoCanvas(state); + canvas.onTogglePlay = () => this.togglePlay(); + canvas.onStopped = () => this.syncPlayButton(); + this.mount(canvas, [this.playButton]); + state.on('load', () => this.canvas.stop()); + state.on('selection', () => this.refresh()); + } + + override selectedId(): string { + return this.state.selectedVideoId ?? ''; + } + + override isMine(asset: Asset): asset is VideoAsset { + return asset.type === 'video'; + } + + get isPlaying(): boolean { + return this.canvas.isPlaying; + } + + togglePlay(): void { + if (this.canvas.isPlaying) this.canvas.stop(); + else this.canvas.play(); + this.syncPlayButton(); + } + + /** Only called while the tab is visible. */ + tick(): void { + this.canvas.tick(); + } + + protected override closed(assetId: string): void { + if (this.canvas.currentAssetId === assetId) this.canvas.stop(); + } + + protected override infoText(asset: VideoAsset): string { + if (!asset.file) return 'Not uploaded'; + const span = this.canvas.shownSpan(); + const whole = formatDuration(asset.duration); + const length = + span && span.end - span.start < asset.duration + ? `${formatDuration(span.end - span.start)} of ${whole}` + : whole; + return `${asset.width} × ${asset.height} · ${length} · ${Number(asset.fps.toFixed(2))} fps`; + } + + protected override syncExtras(asset: VideoAsset | null): void { + this.fitButton.setEnabled(Boolean(asset?.file)); + this.syncPlayButton(); + } + + private syncPlayButton(): void { + const playing = this.canvas.isPlaying; + this.playButton.setIcon(playing ? TOOLBAR_ICON_SVG.stop : TOOLBAR_ICON_SVG.play); + this.playButton.setText(playing ? 'Stop' : 'Play'); + this.playButton.setEnabled(Boolean(this.asset()?.file)); + } +} diff --git a/src/editor/video/VideoPicker.ts b/src/editor/video/VideoPicker.ts new file mode 100644 index 00000000..0fd6f9ab --- /dev/null +++ b/src/editor/video/VideoPicker.ts @@ -0,0 +1,185 @@ +import { Container, Graphics, Sprite, type Text } from 'pixi.js'; +import type { StoredVideo } from '../../core/store/VideoStore'; +import { crispText, formatBytes, textStyle, truncate } from '../../ui'; +import { posterLibrary } from '../../viewport/VideoLibrary'; +import { pickFiles } from '../assetImport'; +import { + currentPicker, + FilePicker, + openPicker, + type PickerItem, + type PickerRow, + RowList, + withAssetNames, +} from '../FilePicker'; +import { containRect } from '../texture/imageGrid'; +import { importStoredVideo, uploadVideos, VIDEO_ACCEPT, type VideoImportDeps } from '../videoImport'; +import { isPending, videoStatusText } from './videoInfo'; + +const ROW_HEIGHT = 52; +const POSTER_W = 80; +const POSTER_H = 45; +const ROW_PAD = 8; +/** How often the list is asked again while a video is on its way. */ +const POLL_MS = 1000; + +type VideoItem = PickerItem; +interface VideoRow extends PickerRow { + poster: Sprite; + frame: Graphics; + detail: Text; +} + +/** One row per video: its poster, its name, and what it is or how far its conversion is. */ +class VideoList extends RowList { + protected readonly rowHeight = ROW_HEIGHT; + + constructor() { + super(); + this.watch(posterLibrary, () => this.redraw()); + } + + protected buildRow(): VideoRow { + const box = new Container(); + const background = new Graphics(); + const frame = new Graphics(); + const poster = new Sprite(); + const caption = crispText({ style: textStyle('body') }); + const detail = crispText({ style: textStyle('small') }); + box.addChild(background, frame, poster, caption, detail); + return { box, background, frame, poster, caption, detail }; + } + + protected redrawRow(row: VideoRow, item: VideoItem, selected: boolean): void { + this.drawBox(row, this.w, ROW_HEIGHT, selected); + const top = (ROW_HEIGHT - POSTER_H) / 2; + row.frame.clear().rect(ROW_PAD, top, POSTER_W, POSTER_H).fill(0x000000); + const texture = item.status === 'ready' && item.poster ? posterLibrary.load(item.poster) : null; + row.poster.visible = texture !== null; + if (texture) { + const fitted = containRect(texture.width, texture.height, POSTER_W, POSTER_H); + row.poster.texture = texture; + row.poster.position.set(ROW_PAD + fitted.x, top + fitted.y); + row.poster.width = fitted.w; + row.poster.height = fitted.h; + } + + const left = ROW_PAD * 2 + POSTER_W; + const width = this.w - left - ROW_PAD; + row.caption.style = textStyle(item.assetName ? 'accent' : 'body'); + row.caption.text = truncate(item.assetName ?? (item.name || 'Video'), row.caption.style, width); + row.caption.position.set(left, 8); + row.detail.style = textStyle(item.status === 'failed' ? 'danger' : 'small'); + row.detail.text = truncate(videoStatusText(item), row.detail.style, width); + row.detail.position.set(left, ROW_HEIGHT - 20); + } +} + +/** + * "+ → Video" (#402). An upload is converted on the server before it can be used, so the list + * shows the ones on their way with their progress and asks again until none is left. What was + * uploaded from here is made an asset the moment it is ready. + */ +class VideoPicker extends FilePicker { + declare protected readonly list: VideoList; + private poll: ReturnType | null = null; + private lastItems: VideoItem[] = []; + + constructor( + private readonly deps: VideoImportDeps, + /** Uploads to adopt once they are ready; the dialog closes when the last one is. */ + private readonly waiting: Set, + ) { + super( + new VideoList(), + { + title: 'Choose a video', + loading: 'Loading videos…', + idle: 'Videos are converted to MP4 on the server, up to 1080p.', + empty: 'No videos on the server yet – upload one.', + use: 'Use video', + noun: 'video', + }, + () => uploadFromDialog(deps), + (file) => deps.videos.delete(file), + ); + } + + /** Global centre of a row (test hook). */ + rowGlobal(file: string): { x: number; y: number } | null { + return this.list.rowGlobal(file); + } + + /** The uploads it waits for (test hook). */ + get waitingFor(): string[] { + return [...this.waiting]; + } + + override close(): void { + if (this.poll !== null) clearTimeout(this.poll); + this.poll = null; + super.close(); + } + + override async load(): Promise { + await super.load(); + if (this.list.destroyed) return; + // What was just uploaded is what the dialog is open for: shown with its progress. + const first = [...this.waiting][0]; + if (!this.list.selected && first) this.list.select(first); + const items = this.list.fileList.map((file) => this.itemOf(file)).filter((item) => item !== null); + for (const item of items) { + if (!this.waiting.has(item.file) || isPending(item)) continue; + this.waiting.delete(item.file); + if (item.status === 'ready') importStoredVideo(item, this.deps); + else this.list.select(item.file); + if (this.waiting.size === 0 && item.status === 'ready') { + this.close(); + return; + } + } + if (items.some(isPending)) this.poll = setTimeout(() => void this.load(), POLL_MS); + } + + protected async fetchItems(): Promise { + const items = withAssetNames(await this.deps.videos.list(), this.deps.state.project, 'video'); + this.lastItems = items; + return items; + } + + protected contentHeight(count: number): { unit: number; total: number } { + return { unit: ROW_HEIGHT, total: this.list.heightOfRows(count) }; + } + + protected adopt(item: VideoItem): void { + importStoredVideo(item, this.deps); + } + + protected override canUse(item: VideoItem): boolean { + return item.status === 'ready'; + } + + protected override describe(item: VideoItem): string { + const size = item.status === 'ready' ? ` – ${formatBytes(item.bytes)}` : ''; + return `${item.name || 'Video'}${size} – ${videoStatusText(item)}`; + } + + private itemOf(file: string): VideoItem | null { + return this.lastItems.find((item) => item.file === file) ?? null; + } +} + +/** The dialog's Upload… button: the file dialog, the uploads, then the picker again to watch them. */ +function uploadFromDialog(deps: VideoImportDeps): void { + void pickFiles(VIDEO_ACCEPT) + .then((files) => uploadVideos(files, deps, (pending) => openVideoPicker(deps, pending))) + .catch((error: unknown) => deps.state.setNotice(String(error), 'error')); +} + +export function openVideoPicker(deps: VideoImportDeps, waiting: readonly string[] = []): VideoPicker { + return openPicker(new VideoPicker(deps, new Set(waiting))); +} + +export function currentVideoPicker(): VideoPicker | null { + return currentPicker(VideoPicker); +} diff --git a/src/editor/video/videoInfo.ts b/src/editor/video/videoInfo.ts new file mode 100644 index 00000000..944bc7a8 --- /dev/null +++ b/src/editor/video/videoInfo.ts @@ -0,0 +1,36 @@ +import type { UploadedVideo } from '../../core/store/VideoStore'; + +/** "12.5 s", "1:05", "1:02:03": as long as it needs to be. */ +export function formatDuration(seconds: number): string { + const s = Math.max(0, seconds); + if (s < 60) return `${Number(s.toFixed(1))} s`; + const whole = Math.round(s); + const h = Math.floor(whole / 3600); + const m = Math.floor((whole % 3600) / 60); + const rest = String(whole % 60).padStart(2, '0'); + return h > 0 ? `${h}:${String(m).padStart(2, '0')}:${rest}` : `${m}:${rest}`; +} + +/** What a video is, or where it is on its way there (#402) – the picker's second line. */ +export function videoStatusText(video: UploadedVideo): string { + switch (video.status) { + case 'queued': + return 'Waiting to be converted…'; + case 'converting': + return `Converting… ${Math.round((video.progress ?? 0) * 100)} %`; + case 'failed': + return `Could not be converted: ${video.error || 'no reason given'}`; + default: + return [ + `${video.width ?? 0} × ${video.height ?? 0}`, + formatDuration(video.duration ?? 0), + `${Number((video.fps ?? 0).toFixed(2))} fps`, + video.audio ? 'with sound' : 'silent', + ].join(' · '); + } +} + +/** Whether a listed video is still on its way, so the list is asked again. */ +export function isPending(video: UploadedVideo): boolean { + return video.status === 'queued' || video.status === 'converting'; +} diff --git a/src/editor/videoImport.ts b/src/editor/videoImport.ts new file mode 100644 index 00000000..15dea713 --- /dev/null +++ b/src/editor/videoImport.ts @@ -0,0 +1,77 @@ +import { CreateVideoCommand } from '../core/commands/assetCommands'; +import type { UploadedVideo, VideoStore } from '../core/store/VideoStore'; +import { type ImportKind, type ImportOutcome, importStored } from './assetImport'; +import type { EditorState } from './state'; + +/** What the file dialog offers and a drop accepts; ffmpeg on the server has the last word. */ +export const VIDEO_ACCEPT = '.mp4,.mov,.m4v,.webm,.mkv,.avi'; + +const VIDEO_EXTENSION = /\.(mp4|mov|m4v|webm|mkv|avi)$/i; + +export function isVideoFileName(name: string): boolean { + return VIDEO_EXTENSION.test(name); +} + +export interface VideoImportDeps { + state: EditorState; + videos: VideoStore; +} + +/** + * A video becomes an asset only once the worker has made it (#402): the asset copies its size, + * length and frame rate, which are not known before. + */ +const VIDEO_KIND: ImportKind = { + type: 'video', + noun: 'video', + fallbackName: 'Video', + store: (deps) => deps.videos, + prepare: (file) => + isVideoFileName(file.name) + ? Promise.resolve({ blob: file, type: 'application/octet-stream' }) + : Promise.reject(new Error('Not a video file – use MP4, MOV, WebM, MKV or AVI.')), + create: (name, video) => + new CreateVideoCommand(name, video.file, { + width: video.width ?? 0, + height: video.height ?? 0, + duration: video.duration ?? 0, + fps: video.fps ?? 0, + audio: video.audio ?? false, + }), +}; + +export function importStoredVideo(video: UploadedVideo, deps: VideoImportDeps): ImportOutcome { + return importStored(VIDEO_KIND, video, deps); +} + +/** + * Sends the files up one after the other and says how it went; what is done already is made an + * asset at once. The rest is converted on the server – `waiting` gets their names, for the picker + * to watch and adopt when they are ready. + */ +export async function uploadVideos( + files: readonly File[], + deps: VideoImportDeps, + waiting: (files: string[]) => void, +): Promise { + const pending: string[] = []; + const failed: string[] = []; + for (const file of files) { + try { + const { blob, type } = await VIDEO_KIND.prepare(file); + const uploaded = await deps.videos.upload(blob, type, file.name); + if (uploaded.status === 'ready') importStoredVideo(uploaded, deps); + else pending.push(uploaded.file); + } catch (error) { + failed.push(`"${file.name}": ${error instanceof Error ? error.message : String(error)}`); + } + } + if (pending.length > 0) waiting(pending); + if (failed.length > 0) deps.state.setNotice(`Could not upload ${failed.join(', ')}`, 'error'); + else if (pending.length > 0) + deps.state.setNotice( + pending.length === 1 + ? 'Converting the video on the server…' + : `Converting ${pending.length} videos on the server…`, + ); +} diff --git a/src/player/Player.ts b/src/player/Player.ts index 3cf7c195..7f13effd 100644 --- a/src/player/Player.ts +++ b/src/player/Player.ts @@ -7,6 +7,7 @@ import { behaviorError } from '../runtime/systems/BehaviorSystem'; import { Input } from '../runtime/systems/Input'; import { requestRender } from '../ui'; import { soundLibrary } from '../viewport/SoundLibrary'; +import { videoLibrary } from '../viewport/VideoLibrary'; import type { GameScreen } from './GameScreen'; /** @@ -57,6 +58,7 @@ export class Player { clock: gameClock, saves: this.saves, }); + videoLibrary.startRun(); this.input.attach(); this.screen.setWorld(this.run.world); if (this.run.world.stopRequested) this.stop(); @@ -68,6 +70,7 @@ export class Player { this.run = null; this.input.detach(); soundLibrary.stopAll(); + videoLibrary.endRun(); gameClock.reset(); const error = behaviorError(run.world); if (error) console.error('[game]', error); diff --git a/src/viewport/SoundLibrary.ts b/src/viewport/SoundLibrary.ts index bfcb2e07..6553d91f 100644 --- a/src/viewport/SoundLibrary.ts +++ b/src/viewport/SoundLibrary.ts @@ -1658,6 +1658,20 @@ export class SoundLibrary implements AudioSink { return this.live()!; } + /** + * The sound of a video (#402) into the master, as a voice goes: from now on `element` is heard + * through the mix and only through it. The gain is its volume. Null while a bake has the context. + */ + mediaInput(element: HTMLMediaElement): GainNode | null { + this.audio(); + const context = this.live(); + if (!context) return null; + if (context.state === 'suspended' && !this.held) void context.resume(); + const gain = context.createGain(); + context.createMediaElementSource(element).connect(gain).connect(this.busInput('')); + return gain; + } + /** The context it plays to the speakers on; null before there is one, and for a bake (#340). */ private live(): AudioContext | null { return this.render ? null : (this.context as AudioContext | null); diff --git a/src/viewport/TextureLibrary.ts b/src/viewport/TextureLibrary.ts index 9ea7ac02..d66865fc 100644 --- a/src/viewport/TextureLibrary.ts +++ b/src/viewport/TextureLibrary.ts @@ -1,9 +1,11 @@ import { Assets, CanvasSource, Texture } from 'pixi.js'; import { isGeneratedFile } from '../core/model/generated'; +import { isVideoKey } from '../core/model/video'; import type { FrameRect, TextureRef } from '../core/model/texture'; import { type EdgeMode, SHEET_EDGE } from '../core/shader/nodes'; import { FileLibrary } from './FileLibrary'; import { GeneratedPictures, type PictureFactory } from './generatedPictures'; +import { videoLibrary } from './VideoLibrary'; /** * Pixi turns `clamp-to-edge` into `repeat` on every texture a Graphics fill draws with @@ -45,6 +47,7 @@ export class TextureLibrary extends FileLibrary { constructor() { super('/game-maker/data/textures', 'textures'); + videoLibrary.onChange(() => this.notifyChange()); } /** Installs who draws generated pictures; without one, a generated key stays empty. */ @@ -72,6 +75,8 @@ export class TextureLibrary extends FileLibrary { get(ref: TextureRef, edge: EdgeMode = SHEET_EDGE): Texture | null { const sheet = this.load(ref.file); if (!sheet) return null; + // A video (#402) is a picture that changes every frame; a cut of it would stand still. + if (isVideoKey(ref.file)) return sheet; const rect = ref.rect ?? (edge === SHEET_EDGE ? null : wholeOf(sheet)); const texture = rect ? this.crop(ref.file, sheet, rect, edge) : sheet; if (!texture) return null; @@ -92,6 +97,7 @@ export class TextureLibrary extends FileLibrary { */ load(file: string): Texture | null { if (isGeneratedFile(file)) return this.generated.get(file); + if (isVideoKey(file)) return videoLibrary.get(file); const texture = this.loaded.get(file); if (!texture) { this.startLoading(file); @@ -103,6 +109,7 @@ export class TextureLibrary extends FileLibrary { /** The picture, or one of its cut frames, if in memory – without a load or a scale-mode change. */ peek(file: string, rect: FrameRect | null = null): Texture | null { if (rect) return this.crops.get(cropKey(file, rect)) ?? null; + if (isVideoKey(file)) return videoLibrary.get(file); return this.loaded.get(file) ?? this.generated.peek(file); } @@ -140,7 +147,7 @@ export class TextureLibrary extends FileLibrary { * loads, and for a generated picture, which lives on the GPU only. */ pixels(file: string): ImageData | null { - const sheet = isGeneratedFile(file) ? null : this.load(file); + const sheet = isGeneratedFile(file) || isVideoKey(file) ? null : this.load(file); const picture = sheet?.source.resource as CanvasImageSource | null | undefined; if (!sheet || !picture) return null; const { pixelWidth: w, pixelHeight: h } = sheet.source; diff --git a/src/viewport/VideoLibrary.ts b/src/viewport/VideoLibrary.ts new file mode 100644 index 00000000..1bd6305f --- /dev/null +++ b/src/viewport/VideoLibrary.ts @@ -0,0 +1,259 @@ +/** + * The players behind every video a sprite wears (#402): one `