Let a tilemap layer sort its rows by Y with the objects (#228)

A tilemap layer was one piece in the draw order, so a figure stood
either in front of a whole layer of trees or behind all of it. For a
top-down game the trees in front of the figure's feet have to cover
it and the ones behind must not.

- A new "Sort by Y" box under the layer list sets `sortByY` on the
  painted layer. Every change is its own undo step.
- Such a layer comes as one draw slot per row. On its `z`, the rows
  and all objects on that `z` are sorted by where they stand: the
  bottom of an object's box, the bottom of a row. Ties keep the
  hierarchy order, and other `z` values are untouched.
- A layer drawn as one piece and the map's own node stay behind the
  sorted ones on the same `z`, since neither has a single place to
  stand. Ground belongs on a lower `z` or stays unsorted.
- A new tile property "Sort offset" (whole rows, -16 to 16) lets a
  crown sort with the trunk below it.
- `TilemapField` draws a container per row for such a layer, and
  picking takes the row into account, so a click picks what is drawn
  in front. Scene view, game view and play share the same rule.

Limits: on a rotated map "down" is world Y. A layer sorted by Y costs
about one draw call per row and tile instead of one per tile.

Tests:
- ./check.sh: 160 files, 2350 tests, CHECK EXIT 0 (before 2340).
- 10 new unit tests in tests/viewport/sort-by-y.test.ts.
- e2e 39-43: ALL PASSED, 148 checks. New check 40p-iii ticks the box
  and finds the slot of row 0, undo brings back the whole layer.
- Counter checks: without the Y sort 5 unit tests fail, with the
  offset ignored 5, with no row in picking 1. If TilemapField draws
  no rows, exactly 40p-iii fails in e2e.
- The e2e sum in the README is 1378, counted, not yet seen in CI.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/viewport/drawOrder.ts

@@ -1,30 +1,99 @@
 import { isSymbolType } from '../core/model/AssetTypes';
 import { sortByZ } from '../core/model/Properties';
-import type { DrawItem } from './items';
+import { cellOrigin, type TileDraw, tilemapTileSize } from '../core/model/tilemap';
+import { footprintExtent } from '../core/properties/bounds';
+import type { DrawItem, TilemapDraw, TilemapLayerDraw } from './items';
 
 /**
  * One thing sorted into the draw order: an object, or one layer of a placed tilemap (#174) –
  * `layerId` is empty for the object itself. A layer is sorted on its own, so a figure can walk
- * between the ground and the treetops of one map.
+ * between the ground and the treetops of one map. A layer sorted by Y comes as one slot per row
+ * (#228), `row` naming it.
  */
 export interface DrawSlot {
   id: string;
   layerId: string;
+  row: number | null;
   z: number;
 }
 
+/** What names a slot, for the viewport's containers and for picking. */
+export function slotKey(id: string, layerId: string, row: number | null): string {
+  return `${id}#${layerId}#${row ?? ''}`;
+}
+
+/** The row a cell is sorted by on a layer sorted by Y: its own, moved down by its tile's offset. */
+export function sortRowOf(draw: TilemapDraw, tile: TileDraw): number {
+  return tile.y + (draw.tiles.get(tile.tileId)?.sortOffset ?? 0);
+}
+
+/** The cells of a layer sorted by Y, grouped by the row they sort by, top row first. */
+export function layerRows(draw: TilemapDraw, layer: TilemapLayerDraw): Map<number, TileDraw[]> {
+  const rows = new Map<number, TileDraw[]>();
+  for (const tile of layer.tiles) {
+    const row = sortRowOf(draw, tile);
+    const held = rows.get(row);
+    if (held) held.push(tile);
+    else rows.set(row, [tile]);
+  }
+  return new Map([...rows].sort(([a], [b]) => a - b));
+}
+
+interface Candidate extends DrawSlot {
+  item: DrawItem;
+}
+
 /**
  * Back to front: ascending `z`, ties in hierarchy order, and a map's layers right after the map in
  * their list order. With every layer at 0 that is exactly the order of before, where a map was one
  * piece. Symbols are left out, they are drawn above everything.
+ *
+ * Where a layer sorted by Y sits on a `z`, everything on that `z` is sorted by where it stands
+ * (#228): an object by the bottom of its box, a row of the layer by the bottom of the row. Ties
+ * keep the order above. A layer drawn as one piece and the map's own node stay behind them, in
+ * that order, since neither has a single place to stand.
  */
 export function drawSlots(items: readonly DrawItem[]): DrawSlot[] {
-  const slots: DrawSlot[] = [];
+  const slots: Candidate[] = [];
   for (const item of items) {
     if (isSymbolType(item.type)) continue;
-    slots.push({ id: item.id, layerId: '', z: item.z });
-    for (const layer of item.tilemap?.layers ?? [])
-      slots.push({ id: item.id, layerId: layer.id, z: item.z + layer.z });
+    slots.push({ id: item.id, layerId: '', row: null, z: item.z, item });
+    for (const layer of item.tilemap?.layers ?? []) {
+      const z = item.z + layer.z;
+      if (!layer.sortByY) slots.push({ id: item.id, layerId: layer.id, row: null, z, item });
+      else
+        for (const row of layerRows(item.tilemap!, layer).keys())
+          slots.push({ id: item.id, layerId: layer.id, row, z, item });
+    }
   }
-  return sortByZ(slots);
+  const sorted = sortByZ(slots);
+  const byY = new Set(sorted.filter((slot) => slot.row !== null).map((slot) => slot.z));
+  const order: Candidate[] = [];
+  for (let start = 0; start < sorted.length;) {
+    const z = sorted[start]!.z;
+    let end = start;
+    while (end < sorted.length && sorted[end]!.z === z) end++;
+    const group = sorted.slice(start, end);
+    if (!byY.has(z)) order.push(...group);
+    else {
+      const standing = group.filter(standsOnY);
+      order.push(...group.filter((slot) => !standsOnY(slot)));
+      const feet = new Map(standing.map((slot) => [slot, feetOf(slot)]));
+      order.push(...standing.sort((a, b) => feet.get(a)! - feet.get(b)!));
+    }
+    start = end;
+  }
+  return order.map(({ id, layerId, row, z }) => ({ id, layerId, row, z }));
+}
+
+/** A row of a layer sorted by Y, or an object other than a map. */
+function standsOnY(slot: Candidate): boolean {
+  return slot.row !== null || (slot.layerId === '' && slot.item.type !== 'tilemap');
+}
+
+/** Where a slot stands, in world Y: the bottom of an object's box, or of a map's row. */
+function feetOf({ item, row }: Candidate): number {
+  if (row === null) return footprintExtent(item.type, item.props, item.m, item.frame).bottom;
+  const bottom = cellOrigin(item.props, 0, row).y + tilemapTileSize(item.props);
+  return item.m.ty + bottom * item.m.d;
 }

src/viewport/items.ts

@@ -5,7 +5,14 @@ import { resolvedHolder } from '../core/model/instance';
 import type { Mat } from '../core/model/matrix';
 import type { PropertyBag } from '../core/model/Properties';
 import type { ParticlesAsset, TilemapAsset } from '../core/model/Asset';
-import { readTile, tileFps, type TileInfo, type TilePlayback, tilePlayback } from '../core/model/tile';
+import {
+  readTile,
+  tileFps,
+  type TileInfo,
+  type TilePlayback,
+  tilePlayback,
+  tileSortOffset,
+} from '../core/model/tile';
 import { isLayerVisible, layerTiles, layerZ, type TileDraw } from '../core/model/tilemap';
 import {
   type CellPolygon,
@@ -66,15 +73,19 @@ export interface TilemapTile {
   info: TileInfo;
   playback: TilePlayback;
   fps: number;
+  /** Rows further down its cells are sorted on a layer sorted by Y (#228). */
+  sortOffset: number;
 }
 
 /**
  * One layer, already worked out: which quarters each cell shows and which tile draws it, and how
- * far above the map's own `z` it is sorted in among the other objects (#174).
+ * far above the map's own `z` it is sorted in among the other objects (#174) – as one piece, or
+ * row by row by Y (#228).
  */
 export interface TilemapLayerDraw {
   id: string;
   z: number;
+  sortByY: boolean;
   tiles: TileDraw[];
 }
 
@@ -133,7 +144,7 @@ function resolveTilemap(
   for (const layer of asset.layers) {
     if (!isLayerVisible(layer)) continue;
     const tiles = layerTiles(project, props, layer);
-    layers.push({ id: layer.id, z: layerZ(layer), tiles });
+    layers.push({ id: layer.id, z: layerZ(layer), sortByY: layer.sortByY === true, tiles });
     for (const tile of tiles) {
       if (used.has(tile.tileId)) continue;
       const source = findAssetOf(project, tile.tileId, 'tile');
@@ -142,6 +153,7 @@ function resolveTilemap(
         info: readTile(project, source),
         playback: tilePlayback(source),
         fps: tileFps(source),
+        sortOffset: tileSortOffset(source),
       });
     }
   }

src/viewport/pick.ts

@@ -1,11 +1,15 @@
 import { isSymbolType } from '../core/model/AssetTypes';
-import { drawSlots } from './drawOrder';
+import { drawSlots, slotKey } from './drawOrder';
 import type { DrawItem } from './items';
 
-/** An object whose hit area is under the pointer; `layers` names a tilemap's layers with a tile there. */
+/**
+ * An object whose hit area is under the pointer; `layers` names a tilemap's layers with a tile
+ * there, and `rows` the row that tile sorts by on each of them sorted by Y (#228).
+ */
 export interface PointerHit {
   id: string;
   layers: readonly string[] | null;
+  rows?: ReadonlyMap<string, number>;
 }
 
 /**
@@ -16,7 +20,7 @@ export interface PointerHit {
  */
 export function stackAt(items: readonly DrawItem[], hits: readonly PointerHit[]): string[] {
   const rank = new Map<string, number>();
-  drawSlots(items).forEach((slot, i) => rank.set(`${slot.id}#${slot.layerId}`, i));
+  drawSlots(items).forEach((slot, i) => rank.set(slotKey(slot.id, slot.layerId, slot.row), i));
   const symbolsFrom = rank.size;
   const index = new Map(items.map((item, i) => [item.id, i]));
   const ranked: { id: string; rank: number }[] = [];
@@ -25,7 +29,9 @@ export function stackAt(items: readonly DrawItem[], hits: readonly PointerHit[])
     if (at === undefined) continue;
     const ranks = isSymbolType(items[at]!.type)
       ? [symbolsFrom + at]
-      : (hit.layers ?? ['']).map((layerId) => rank.get(`${hit.id}#${layerId}`) ?? -1);
+      : (hit.layers ?? ['']).map(
+          (layerId) => rank.get(slotKey(hit.id, layerId, hit.rows?.get(layerId) ?? null)) ?? -1,
+        );
     const best = Math.max(-1, ...ranks);
     if (best >= 0) ranked.push({ id: hit.id, rank: best });
   }

src/viewport/tilemap.ts

@@ -6,7 +6,8 @@
  */
 import { Container, Mesh, MeshGeometry, type Texture } from 'pixi.js';
 import { pictureIndexAt, type TilePicture } from '../core/model/tile';
-import { cellOrigin, tilemapTileSize } from '../core/model/tilemap';
+import { cellOrigin, type TileDraw, tilemapTileSize } from '../core/model/tilemap';
+import { layerRows } from './drawOrder';
 import type { DrawItem, TilemapTile } from './items';
 import { placeNode } from './symbols';
 import type { TextureLibrary } from './TextureLibrary';
@@ -30,6 +31,7 @@ export function layerAlpha(layerId: string, focus: LayerFocus | null): number {
 /**
  * One container per layer rather than one per map: each layer has a `z` of its own (#174) and is
  * sorted in among the other objects on its own – where is the viewport's call, not this one's.
+ * A layer sorted by Y has one container per row instead (#228), keyed like its draw slot.
  */
 export class TilemapField {
   private readonly layers = new Map<string, Container>();
@@ -49,9 +51,9 @@ export class TilemapField {
     for (const layer of this.layers.values()) layer.visible = shown;
   }
 
-  /** The container a layer draws into; null for a layer the last sync did not draw. */
-  layerContainer(layerId: string): Container | null {
-    return this.layers.get(layerId) ?? null;
+  /** The container a layer, or one row of it, draws into; null for one the last sync did not draw. */
+  layerContainer(layerId: string, row: number | null = null): Container | null {
+    return this.layers.get(rowKey(layerId, row)) ?? null;
   }
 
   /**
@@ -67,27 +69,37 @@ export class TilemapField {
     const alive = new Set<string>();
     const aliveLayers = new Set<string>();
     for (const layer of draw?.layers ?? []) {
-      aliveLayers.add(layer.id);
-      // Each layer sits under a node of its own carrying the map's transform, so that is applied
-      // once per layer rather than baked into every vertex.
-      const container = this.layerFor(item.id, layer.id);
-      placeNode(container, item.m);
+      const parts: [number | null, TileDraw[]][] =
+        draw && layer.sortByY ? [...layerRows(draw, layer)] : [[null, layer.tiles]];
       const alpha = layerAlpha(layer.id, focus);
-      if (alpha === 0) continue;
-      for (const [tileId, tiles] of groupByTile(layer.tiles)) {
-        const source = draw?.tiles.get(tileId);
-        const picture = source && pictureNow(source, now);
-        const texture = picture ? textures.get(picture.ref) : null;
-        if (!source || !picture || !texture) continue;
-        if (source.info.pictures.length > 1) this.animated = true;
-        const key = `${layer.id}|${tileId}`;
-        alive.add(key);
-        const { positions, uvs, indices } = tileMesh(tiles, tile, origin, sheetOf(source, picture, texture));
-        const mesh = this.meshFor(key, new MeshGeometry({ positions, uvs, indices }), texture);
-        // Per mesh rather than per layer: the meshes of one layer never overlap, so fading each
-        // one looks the same as fading the layer as a whole.
-        mesh.alpha = alpha;
-        drawn.push({ layer: container, mesh });
+      for (const [row, cells] of parts) {
+        const part = rowKey(layer.id, row);
+        aliveLayers.add(part);
+        // Each part sits under a node of its own carrying the map's transform, so that is applied
+        // once per part rather than baked into every vertex.
+        const container = this.layerFor(item.id, part);
+        placeNode(container, item.m);
+        if (alpha === 0) continue;
+        for (const [tileId, tiles] of groupByTile(cells)) {
+          const source = draw?.tiles.get(tileId);
+          const picture = source && pictureNow(source, now);
+          const texture = picture ? textures.get(picture.ref) : null;
+          if (!source || !picture || !texture) continue;
+          if (source.info.pictures.length > 1) this.animated = true;
+          const key = `${part}|${tileId}`;
+          alive.add(key);
+          const { positions, uvs, indices } = tileMesh(
+            tiles,
+            tile,
+            origin,
+            sheetOf(source, picture, texture),
+          );
+          const mesh = this.meshFor(key, new MeshGeometry({ positions, uvs, indices }), texture);
+          // Per mesh rather than per layer: the meshes of one layer never overlap, so fading each
+          // one looks the same as fading the layer as a whole.
+          mesh.alpha = alpha;
+          drawn.push({ layer: container, mesh });
+        }
       }
     }
     for (const [key, mesh] of this.meshes) {
@@ -106,15 +118,15 @@ export class TilemapField {
     this.drawCalls = drawn.length;
   }
 
-  private layerFor(objectId: string, layerId: string): Container {
-    let layer = this.layers.get(layerId);
+  private layerFor(objectId: string, part: string): Container {
+    let layer = this.layers.get(part);
     if (!layer) {
       layer = new Container();
       layer.eventMode = 'none';
       // The draw order reports it by this, next to the object's own id.
-      layer.label = `${objectId}#${layerId}`;
+      layer.label = `${objectId}#${part}`;
       layer.visible = this.shown;
-      this.layers.set(layerId, layer);
+      this.layers.set(part, layer);
     }
     return layer;
   }
@@ -140,6 +152,11 @@ export class TilemapField {
   }
 }
 
+/** A whole layer is keyed by its id, as before; a row of a layer sorted by Y by id and row. */
+function rowKey(layerId: string, row: number | null): string {
+  return row === null ? layerId : `${layerId}#${row}`;
+}
+
 /**
  * The block this tile shows right now. A quarter is cut from it by uvs rather than by the
  * texture library, so what is wanted here is the picture of the step and nothing finer.