Tilemap cells are stored compactly in the project file: each layer keeps a tile list, an origin and one run-length encoded string per row; older projects with a cell map still load (#134)

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

README.md

@@ -38,7 +38,7 @@ npm run build        # writes dist/ and regenerates index.html; the folder is wh
 The browser suite and the three example games need a machine with a GPU:
 
 ```bash
-node tools/seed-e2e.mjs && node tools/e2e.mjs      # the editor, 1013 checks (GM_STEPS=20 for one step)
+node tools/seed-e2e.mjs && node tools/e2e.mjs      # the editor, 1017 checks (GM_STEPS=20 for one step)
 node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs
 node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs
 node tools/seed-actor.mjs && node tools/smoke-actor.mjs

src/core/model/tilemap.ts

@@ -6,6 +6,7 @@
  * Cells are sparse – a map from `"x,y"` to a tile id – rather than one entry per grid square.
  * A map is what painting, erasing and reading a neighbour all want, and changing the grid size then
  * moves nothing: what falls outside is simply not drawn and comes back when the grid grows again.
+ * The project file stores them more compactly (#134, `store/tilemapCells.ts`).
  */
 import type { TileAsset, TilemapLayer } from './Asset';
 import { type Neighbours, type TileQuadrants, tileQuadrants } from './autotile';

src/core/store/serialize.ts

@@ -28,6 +28,7 @@ import type {
   TilemapLayer,
 } from '../model/Asset';
 import { parseCellKey } from '../model/tilemap';
+import { decodeCells, encodeCells, isStoredCells } from './tilemapCells';
 import { isAssetType } from '../model/AssetTypes';
 import { isFontFile } from '../model/font';
 import { isSoundFile } from '../model/sound';
@@ -218,9 +219,19 @@ export function deserializeProject(raw: unknown, fallbackId = ''): Project {
   return project;
 }
 
-/** Plain JSON-ready copy of the project (deep clone, stable key order for readable diffs). */
+/**
+ * Plain JSON-ready copy of the project (deep clone, stable key order for readable diffs). The cells
+ * of a tilemap are the one thing written differently from how they are held: see `tilemapCells.ts`.
+ */
 export function serializeProject(project: Project): unknown {
-  return JSON.parse(JSON.stringify(project));
+  const copy = JSON.parse(JSON.stringify(project)) as Project;
+  for (const asset of copy.assets)
+    if (asset.type === 'tilemap')
+      (asset as { layers: unknown[] }).layers = asset.layers.map(({ cells, ...layer }) => ({
+        ...layer,
+        ...encodeCells(cells),
+      }));
+  return copy;
 }
 
 function normalizeObjects(
@@ -513,8 +524,10 @@ function normalizeLayers(raw: unknown): TilemapLayer[] {
   for (const item of Array.isArray(raw) ? raw : []) {
     const l = asRecord(item);
     if (typeof l.id !== 'string' || !l.id || layers.some((o) => o.id === l.id)) continue;
+    // Projects saved before #134 hold the map itself; it is taken over as it is.
+    const stored = isStoredCells(l) ? decodeCells(l) : asRecord(l.cells);
     const cells: Record<string, string> = {};
-    for (const [key, value] of Object.entries(asRecord(l.cells)))
+    for (const [key, value] of Object.entries(stored))
       if (parseCellKey(key) && typeof value === 'string' && value !== '') cells[key] = value;
     const layer: TilemapLayer = { id: l.id, name: typeof l.name === 'string' ? l.name : 'Layer', cells };
     if (l.hidden === true) layer.hidden = true;

src/core/store/tilemapCells.ts

@@ -0,0 +1,120 @@
+/**
+ * How the cells of a tilemap layer are written to the project file (#134). In memory a layer keeps
+ * its sparse map from `"x,y"` to a tile id – what painting and reading a neighbour want – but in the
+ * file every painted cell would then carry its own key and the full tile id, and the server writes
+ * the file pretty-printed, one entry per line.
+ *
+ * So the file holds, per layer:
+ * - `tiles`: every tile id the layer names, once, in the order they first appear;
+ * - `origin`: the top-left cell of the smallest rectangle around all painted cells;
+ * - `rows`: one string per row of that rectangle, as runs `index*count` (a single cell drops the
+ *   count), separated by spaces. `index` counts from 1 into `tiles`, `0` is an empty cell, and
+ *   empty cells at the end of a row are left out.
+ *
+ * A row of forty grass cells, three empty ones and a single path cell reads `1*40 0*3 2`. The
+ * rectangle is taken from the cells, not from the grid, so cells outside the grid – kept so that
+ * they come back when the grid grows again – are stored as they are.
+ */
+import { cellKey, parseCellKey } from '../model/tilemap';
+
+export interface StoredCells {
+  tiles: string[];
+  origin: { x: number; y: number };
+  rows: string[];
+}
+
+/**
+ * The longest run a row may state. A file is untrusted: `1*999999999` would otherwise build a
+ * billion cells. No grid gets near this, so a real map never meets it.
+ */
+export const MAX_RUN = 65536;
+
+const RUN = /^(\d+)(?:\*(\d+))?$/;
+
+export function encodeCells(cells: Record<string, string>): StoredCells {
+  const painted: { x: number; y: number; id: string }[] = [];
+  for (const [key, id] of Object.entries(cells)) {
+    const at = parseCellKey(key);
+    if (at && id) painted.push({ ...at, id });
+  }
+  if (painted.length === 0) return { tiles: [], origin: { x: 0, y: 0 }, rows: [] };
+  painted.sort((a, b) => a.y - b.y || a.x - b.x);
+
+  const tiles: string[] = [];
+  const index = new Map<string, number>();
+  for (const { id } of painted)
+    if (!index.has(id)) {
+      tiles.push(id);
+      index.set(id, tiles.length);
+    }
+
+  const x0 = painted.reduce((min, c) => Math.min(min, c.x), Infinity);
+  const y0 = painted[0]!.y;
+  const y1 = painted[painted.length - 1]!.y;
+  const rows: string[] = [];
+  let next = 0;
+  for (let y = y0; y <= y1; y++) {
+    const runs: string[] = [];
+    let x = x0;
+    const push = (value: number, count: number): void => {
+      runs.push(count === 1 ? String(value) : `${value}*${count}`);
+    };
+    while (next < painted.length && painted[next]!.y === y) {
+      const cell = painted[next]!;
+      if (cell.x > x) push(0, cell.x - x);
+      const value = index.get(cell.id)!;
+      let count = 1;
+      while (
+        next + count < painted.length &&
+        painted[next + count]!.y === y &&
+        painted[next + count]!.x === cell.x + count &&
+        painted[next + count]!.id === cell.id
+      )
+        count++;
+      push(value, count);
+      x = cell.x + count;
+      next += count;
+    }
+    rows.push(runs.join(' '));
+  }
+  return { tiles, origin: { x: x0, y: y0 }, rows };
+}
+
+/**
+ * The cells a stored layer describes. Read leniently, as the rest of the file is: a run that does
+ * not parse ends its row, because every run after it would land in the wrong column; an index
+ * naming no tile is an empty cell.
+ */
+export function decodeCells(raw: {
+  tiles?: unknown;
+  origin?: unknown;
+  rows?: unknown;
+}): Record<string, string> {
+  const tiles: unknown[] = Array.isArray(raw.tiles) ? raw.tiles : [];
+  const origin = raw.origin && typeof raw.origin === 'object' ? (raw.origin as Record<string, unknown>) : {};
+  const x0 = Number.isSafeInteger(origin.x) ? (origin.x as number) : 0;
+  const y0 = Number.isSafeInteger(origin.y) ? (origin.y as number) : 0;
+  const cells: Record<string, string> = {};
+  const rows = Array.isArray(raw.rows) ? raw.rows : [];
+  rows.forEach((row, r) => {
+    if (typeof row !== 'string') return;
+    let x = x0;
+    for (const token of row.split(/\s+/)) {
+      if (token === '') continue;
+      const match = RUN.exec(token);
+      const value = match ? Number(match[1]) : NaN;
+      const count = match ? Number(match[2] ?? 1) : NaN;
+      if (!Number.isSafeInteger(value) || !(count >= 1 && count <= MAX_RUN)) return;
+      const id = value >= 1 ? tiles[value - 1] : undefined;
+      if (typeof id === 'string' && id !== '')
+        for (let i = 0; i < count; i++) cells[cellKey(x + i, y0 + r)] = id;
+      x += count;
+    }
+  });
+  return cells;
+}
+
+/** Whether a stored layer is in this format rather than the map of the first version (#128). */
+export function isStoredCells(raw: Record<string, unknown>): boolean {
+  return Array.isArray(raw.rows);
+}

tests/core/tilemap-cells.test.ts

@@ -0,0 +1,153 @@
+import { describe, expect, it } from 'vitest';
+import { CreateAssetCommand } from '../../src/core/commands/assetCommands';
+import type { TilemapAsset } from '../../src/core/model/Asset';
+import { findAsset } from '../../src/core/model/Project';
+import { cellKey } from '../../src/core/model/tilemap';
+import { deserializeProject, serializeProject } from '../../src/core/store/serialize';
+import { decodeCells, encodeCells, MAX_RUN } from '../../src/core/store/tilemapCells';
+import { makeProject, roundTrip } from '../helpers';
+
+/** Cells from a picture: one character per cell, `.` is empty, any other names the tile `t<char>`. */
+function cellsFrom(rows: string[], x0 = 0, y0 = 0): Record<string, string> {
+  const cells: Record<string, string> = {};
+  rows.forEach((row, y) =>
+    [...row].forEach((c, x) => {
+      if (c !== '.') cells[cellKey(x0 + x, y0 + y)] = `t${c}`;
+    }),
+  );
+  return cells;
+}
+
+function fullMap(size: number, tileOf: (x: number, y: number) => string): Record<string, string> {
+  const cells: Record<string, string> = {};
+  for (let y = 0; y < size; y++) for (let x = 0; x < size; x++) cells[cellKey(x, y)] = tileOf(x, y);
+  return cells;
+}
+
+/** A project with one tilemap whose single layer holds `cells`, as the editor would have it. */
+function projectWith(cells: Record<string, string>) {
+  const f = makeProject();
+  const create = new CreateAssetCommand('tilemap', 'Ground');
+  f.stack.execute(create);
+  const asset = findAsset(f.project, create.asset.id) as TilemapAsset;
+  asset.layers[0]!.cells = { ...cells };
+  return { project: f.project, id: create.asset.id };
+}
+
+function storedLayer(raw: unknown, id: string): Record<string, unknown> {
+  const assets = (raw as { assets: { id: string; layers?: Record<string, unknown>[] }[] }).assets;
+  return assets.find((a) => a.id === id)!.layers![0]!;
+}
+
+describe('the cells of a tilemap in the file', () => {
+  it('store an empty layer as nothing at all', () => {
+    expect(encodeCells({})).toEqual({ tiles: [], origin: { x: 0, y: 0 }, rows: [] });
+    expect(decodeCells(encodeCells({}))).toEqual({});
+  });
+
+  it('store a single cell as its position and one run', () => {
+    expect(encodeCells({ '7,3': 'grass' })).toEqual({
+      tiles: ['grass'],
+      origin: { x: 7, y: 3 },
+      rows: ['1'],
+    });
+  });
+
+  it('write runs, gaps and several tiles row by row, and read them back', () => {
+    const cells = cellsFrom(['aaab', '..b.', '', 'a..a']);
+    const stored = encodeCells(cells);
+    expect(stored).toEqual({
+      tiles: ['ta', 'tb'],
+      origin: { x: 0, y: 0 },
+      rows: ['1*3 2', '0*2 2', '', '1 0*2 1'],
+    });
+    expect(decodeCells(stored)).toEqual(cells);
+  });
+
+  it('keep cells outside the grid, on either side of it', () => {
+    const cells = { '-3,-2': 'a', '40,-2': 'a', '0,25': 'b' };
+    const stored = encodeCells(cells);
+    expect(stored.origin).toEqual({ x: -3, y: -2 });
+    expect(stored.rows).toHaveLength(28);
+    expect(decodeCells(stored)).toEqual(cells);
+  });
+
+  it('bring a full map with several tiles back exactly', () => {
+    const cells = fullMap(100, (x, y) => (x < 30 ? 'water' : y % 7 === 0 ? 'path' : 'grass'));
+    const stored = encodeCells(cells);
+    // Row 0 is a path row (0 % 7), so path is met before grass.
+    expect(stored.tiles).toEqual(['water', 'path', 'grass']);
+    expect(stored.rows).toHaveLength(100);
+    expect(decodeCells(stored)).toEqual(cells);
+  });
+
+  it('bring a scattered map back exactly', () => {
+    let seed = 7;
+    const random = () => (seed = (seed * 16807) % 2147483647) / 2147483647;
+    const ids = ['a', 'b', 'c', 'd'];
+    const cells: Record<string, string> = {};
+    for (let i = 0; i < 500; i++)
+      cells[cellKey(Math.floor(random() * 60) - 10, Math.floor(random() * 40) - 5)] =
+        ids[Math.floor(random() * ids.length)]!;
+    expect(decodeCells(encodeCells(cells))).toEqual(cells);
+  });
+
+  it('are written compactly by the project file and read back by it', () => {
+    const cells = cellsFrom(['aa.b']);
+    const { project, id } = projectWith(cells);
+    const layer = storedLayer(serializeProject(project), id);
+    expect(layer.cells).toBeUndefined();
+    expect(layer).toMatchObject({ tiles: ['ta', 'tb'], origin: { x: 0, y: 0 }, rows: ['1*2 0 2'] });
+    expect((findAsset(roundTrip(project), id) as TilemapAsset).layers[0]!.cells).toEqual(cells);
+  });
+
+  it('leave the map in memory untouched when saving', () => {
+    const cells = cellsFrom(['ab']);
+    const { project, id } = projectWith(cells);
+    serializeProject(project);
+    expect((findAsset(project, id) as TilemapAsset).layers[0]!.cells).toEqual(cells);
+  });
+
+  it('take a project saved before #134 over as it is, and write it in the new form', () => {
+    const { project, id } = projectWith({});
+    const raw = serializeProject(project) as { assets: Record<string, unknown>[] };
+    const old = raw.assets.find((a) => a.id === id)!;
+    old.layers = [
+      { id: 'L1', name: 'Ground', cells: { '1,2': 'grass', '2,2': 'grass', nonsense: 'x', '3,3': '' } },
+    ];
+    const loaded = deserializeProject(raw);
+    expect((findAsset(loaded, id) as TilemapAsset).layers[0]!.cells).toEqual({
+      '1,2': 'grass',
+      '2,2': 'grass',
+    });
+    expect(storedLayer(serializeProject(loaded), id)).toMatchObject({ tiles: ['grass'], rows: ['1*2'] });
+  });
+
+  it('shrink a full map of 100 × 100 with two layers to a small part of the old size', () => {
+    const ground = fullMap(100, (x) => (x < 50 ? 'grass' : 'dirt'));
+    const decor = fullMap(100, (x, y) => ((x * 7 + y * 3) % 11 === 0 ? 'flower' : 'bush'));
+    const old = JSON.stringify([{ cells: ground }, { cells: decor }], null, 4).length;
+    const now = JSON.stringify([encodeCells(ground), encodeCells(decor)], null, 4).length;
+    expect(now).toBeLessThan(old / 5);
+  });
+});
+
+describe('reading stored cells leniently', () => {
+  it('ends a row at a run that does not parse, since every later run would land wrongly', () => {
+    const cells = decodeCells({ tiles: ['a'], origin: { x: 0, y: 0 }, rows: ['1 x 1', '1*2', '1*-1 1'] });
+    expect(cells).toEqual({ '0,0': 'a', '0,1': 'a', '1,1': 'a' });
+  });
+
+  it('counts an index that names no tile as an empty cell', () => {
+    const cells = decodeCells({ tiles: ['a', 5], origin: { x: 0, y: 0 }, rows: ['3 2 1'] });
+    expect(cells).toEqual({ '2,0': 'a' });
+  });
+
+  it('refuses a run longer than any grid, and a row that is not a string', () => {
+    expect(decodeCells({ tiles: ['a'], rows: [`1*${MAX_RUN + 1}`, 7, '1'] })).toEqual({ '0,2': 'a' });
+  });
+
+  it('starts at the top-left corner when the origin is missing or broken', () => {
+    expect(decodeCells({ tiles: ['a'], origin: { x: 1.5, y: 'no' }, rows: ['0 1'] })).toEqual({ '1,0': 'a' });
+  });
+});

tools/e2e/40-tilemap.mjs

@@ -66,6 +66,7 @@ async function cellShows(t, x, y, quadrants) {
 
 export async function tilemap(t) {
   const { check, json, evaluate, mutate, click, clickText, drag, press, shot, settle, undo, select } = t;
+  const { saved, api, navigate, projectId } = t;
 
   await press('Escape');
   await evaluate('__gm.state.select(null)');
@@ -292,6 +293,33 @@ export async function tilemap(t) {
     const o = __gm.state.activeScene.objects.find((o) => o.id === ${JSON.stringify(MAP_OBJECT)});
     o.position = { x: 0, y: ${LANE}, z: 0 };`);
 
+  // 40k. The file holds the cells compactly (#134): per layer the tiles once and a string of runs
+  //      per row, no key per cell. A reload reads them back into exactly the same cells.
+  const inMemory = await json(
+    `__gm.state.project.assets.find((a) => a.id === ${JSON.stringify(made.id)}).layers.map((l) => l.cells)`,
+  );
+  await saved();
+  const storedMap = (await api(`projects/${projectId}`)).project.assets.find((a) => a.id === made.id);
+  const storedLayers = storedMap?.layers ?? [];
+  check(
+    'the saved map stores each layer as its tiles and a run string per row, not a key per cell',
+    storedLayers.length === 2 &&
+      storedLayers.every((l) => l.cells === undefined && Array.isArray(l.rows)) &&
+      storedLayers[0].tiles.join() === 'at-grass' &&
+      storedLayers[0].rows.every((r) => typeof r === 'string'),
+    JSON.stringify(storedLayers.map(({ tiles, origin, rows }) => ({ tiles, origin, rows }))),
+  );
+  await navigate();
+  const reloaded = await json(
+    `__gm.state.project.assets.find((a) => a.id === ${JSON.stringify(made.id)}).layers.map((l) => l.cells)`,
+  );
+  const sorted = (layers) => JSON.stringify(layers.map((c) => Object.entries(c).sort()));
+  check(
+    'a reloaded project has exactly the cells it was saved with',
+    sorted(reloaded) === sorted(inMemory) && Object.keys(inMemory[0]).length === 9,
+    JSON.stringify({ reloaded, inMemory }),
+  );
+
   // Leave the editor as the rest of the suite expects to find it.
   await evaluate("__gm.state.setTilemapPaint({ tool: 'select' })");
   await evaluate('__gm.state.select(null)');