Add CityGML tile rebasing and pipeline runner

2025-12-18 21:18:47 +01:00
parent d2f24193df
commit af53724ae5
3 changed files with 368 additions and 6 deletions
--- a/README.md
+++ b/README.md
@@ -96,23 +96,44 @@ LoD2 CityGML tiles can be converted to GLB per tile while preserving roof/wall s
   done
   ```
   If cjio still complains about duplicate object IDs, inspect the source GML; `--ignore_duplicate_keys` tells cjio to keep the last occurrence.
-3. Split roof/wall semantics without reindexing vertices: `uv run python scripts/split_semantics.py`
-4. Export GLBs (tile-local coords; place with `export_unity/tile_index.csv`):
+3. Shift coordinates to tile-local XY using the heightmap manifest (Z stays absolute):
   ```bash
-   for f in work/cityjson_tri/*.tri.city.json; do
+   uv run python scripts/rebase_cityjson_tiles.py \
+     --input-dir work/cityjson_tri \
+     --output-dir work/cityjson_tri_local \
+     --tile-index export_unity/tile_index.csv
+   ```
+   If your `tile_index.csv` lives elsewhere, override `--tile-index`. The script uses the numeric tile suffix (e.g., `32_328_5511`) to match LoD2 tiles to the DGM1 manifest.
+4. Split roof/wall semantics without reindexing vertices on the tile-local set:
+   ```bash
+   uv run python scripts/split_semantics.py --input-dir work/cityjson_tri_local --output-dir work/cityjson_split_local
+   ```
+5. Export GLBs (tile-local coords; place with `export_unity/tile_index.csv`):
+   ```bash
+   for f in work/cityjson_tri_local/*.tri.city.json; do
     base="$(basename "$f" .tri.city.json)"
     uv run cjio "$f" export glb "export_unity/buildings_glb/${base}.glb"
   done
-   for f in work/cityjson_split/*.roof.city.json; do
+   for f in work/cityjson_split_local/*.roof.city.json; do
     base="$(basename "$f" .roof.city.json)"
     uv run cjio "$f" export glb "export_unity/buildings_glb_split/${base}_roof.glb"
   done
-   for f in work/cityjson_split/*.wall.city.json; do
+   for f in work/cityjson_split_local/*.wall.city.json; do
     base="$(basename "$f" .wall.city.json)"
     uv run cjio "$f" export glb "export_unity/buildings_glb_split/${base}_wall.glb"
   done
   ```
-5. Confirm intermediates/GLBs exist: `uv run python scripts/verify_pipeline.py --mode both`
+6. Confirm intermediates/GLBs exist (override dirs if using `_local` outputs):
+   ```bash
+   uv run python scripts/verify_pipeline.py --mode both \
+     --tri-dir work/cityjson_tri_local \
+     --split-dir work/cityjson_split_local
+   ```
+
+One-shot test run (assumes `tile_index.csv` is already present from the heightmap export):
+```bash
+bash scripts/run_citygml_to_glb.sh
+```

 ### Troubleshooting
 - Empty raw directories cause VRT creation to fail fast (`No sources available to build VRT`); populate inputs or adjust `--raw-*` overrides.
--- a/scripts/rebase_cityjson_tiles.py
+++ b/scripts/rebase_cityjson_tiles.py
@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+"""Shift CityJSON coordinates into tile-local space using tile_index.csv offsets."""
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import sys
+from pathlib import Path
+from typing import Any, Iterable
+
+DEFAULT_TILE_INDEX = Path("export_unity/tile_index.csv")
+
+
+def parse_args(argv: Iterable[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Rebase CityJSON coordinates so XY are relative to the tile bounds from tile_index.csv."
+    )
+    parser.add_argument(
+        "--input-dir",
+        type=Path,
+        default=Path("work/cityjson_tri"),
+        help="Directory containing CityJSON files (triangulated/split).",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=Path,
+        default=Path("work/cityjson_tri_local"),
+        help="Directory to write tile-local CityJSON files.",
+    )
+    parser.add_argument(
+        "--tile-index",
+        type=Path,
+        default=DEFAULT_TILE_INDEX,
+        help="Path to tile_index.csv produced by the heightmap export.",
+    )
+    parser.add_argument(
+        "--pattern",
+        default="**/*.city.json",
+        help="Glob pattern for input files (defaults to any *.city.json under the input dir).",
+    )
+    return parser.parse_args(argv)
+
+
+def resolve_input_file(path: Path) -> Path | None:
+    """Handle both flat files and citygml-tools style output directories."""
+    if path.is_file():
+        return path
+    if path.is_dir():
+        candidate = path / f"{path.stem}.json"
+        if candidate.is_file():
+            return candidate
+        matches = list(path.glob("*.json"))
+        if len(matches) == 1:
+            return matches[0]
+    return None
+
+
+def strip_suffixes(name: str) -> str:
+    """Remove known suffixes (.tri, .roof, .wall, .ground, .closure, .city.json)."""
+    trimmed = name
+    if trimmed.endswith(".json"):
+        trimmed = trimmed[: -len(".json")]
+    if trimmed.endswith(".city"):
+        trimmed = trimmed[: -len(".city")]
+    for suffix in (".tri", ".roof", ".wall", ".ground", ".closure"):
+        if trimmed.endswith(suffix):
+            trimmed = trimmed[: -len(suffix)]
+    return trimmed
+
+
+def tile_suffix(tile_id: str) -> str:
+    parts = tile_id.split("_")
+    return "_".join(parts[-3:]) if len(parts) >= 3 else tile_id
+
+
+def load_tile_offsets(tile_index: Path) -> dict[str, tuple[float, float]]:
+    if not tile_index.exists():
+        raise SystemExit(f"tile_index.csv missing: {tile_index}")
+
+    offsets: dict[str, tuple[float, float]] = {}
+    with tile_index.open("r", encoding="utf-8", newline="") as handle:
+        reader = csv.DictReader(handle)
+        for row in reader:
+            tile_id = row.get("tile_id")
+            if not tile_id:
+                continue
+            try:
+                xmin = float(row["xmin"])
+                ymin = float(row["ymin"])
+            except (KeyError, TypeError, ValueError):
+                continue
+
+            offset = (xmin, ymin)
+            offsets[tile_id] = offset
+            offsets[tile_suffix(tile_id)] = offset
+    return offsets
+
+
+def read_json(path: Path) -> dict[str, Any]:
+    with path.open("r", encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+def write_json(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as handle:
+        json.dump(payload, handle, ensure_ascii=True, indent=2)
+        handle.write("\n")
+
+
+def ensure_three(values: list[Any] | None, default: float) -> list[float]:
+    resolved = [default, default, default]
+    if not values:
+        return resolved
+
+    for idx, value in enumerate(values[:3]):
+        try:
+            resolved[idx] = float(value)
+        except (TypeError, ValueError):
+            resolved[idx] = default
+    return resolved
+
+
+def compute_extent(vertices: list[list[float]], scale: list[float], translate: list[float]) -> list[float] | None:
+    usable = [vertex for vertex in vertices if len(vertex) >= 3]
+    if not usable:
+        return None
+
+    xs = [vertex[0] * scale[0] + translate[0] for vertex in usable]
+    ys = [vertex[1] * scale[1] + translate[1] for vertex in usable]
+    zs = [vertex[2] * scale[2] + translate[2] for vertex in usable]
+    return [min(xs), min(ys), min(zs), max(xs), max(ys), max(zs)]
+
+
+def rebase_cityjson(cityjson: dict[str, Any], offset: tuple[float, float]) -> None:
+    xmin, ymin = offset
+
+    transform = cityjson.get("transform")
+    if transform:
+        translate = ensure_three(transform.get("translate"), 0.0)
+        scale = ensure_three(transform.get("scale"), 1.0)
+
+        translate[0] -= xmin
+        translate[1] -= ymin
+
+        transform["translate"] = translate
+        transform["scale"] = scale
+        cityjson["transform"] = transform
+    else:
+        for vertex in cityjson.get("vertices") or []:
+            if len(vertex) >= 2:
+                vertex[0] -= xmin
+                vertex[1] -= ymin
+        translate = [0.0, 0.0, 0.0]
+        scale = [1.0, 1.0, 1.0]
+
+    extent = compute_extent(cityjson.get("vertices") or [], scale, translate)
+    if extent:
+        metadata = cityjson.get("metadata") or {}
+        metadata["geographicalExtent"] = extent
+        cityjson["metadata"] = metadata
+
+
+def process_file(path: Path, offsets: dict[str, tuple[float, float]], output_dir: Path) -> int:
+    resolved = resolve_input_file(path)
+    if not resolved:
+        print(f"[skip] cannot resolve CityJSON file for {path}", file=sys.stderr)
+        return 0
+
+    tile_name = strip_suffixes(path.name)
+    offset = offsets.get(tile_name) or offsets.get(tile_suffix(tile_name))
+    if offset is None:
+        print(f"[skip] no tile_index entry for {tile_name}", file=sys.stderr)
+        return 0
+
+    cityjson = read_json(resolved)
+    rebase_cityjson(cityjson, offset)
+
+    output_path = output_dir / path.name
+    write_json(output_path, cityjson)
+    return 1
+
+
+def main(argv: Iterable[str] | None = None) -> int:
+    args = parse_args(argv)
+    offsets = load_tile_offsets(args.tile_index)
+
+    files = sorted(args.input_dir.glob(args.pattern))
+    if not files:
+        print(f"No input files matched pattern '{args.pattern}' in {args.input_dir}", file=sys.stderr)
+        return 1
+
+    written = 0
+    for path in files:
+        written += process_file(path, offsets, args.output_dir)
+
+    print(f"Wrote {written} tile-local file(s) to {args.output_dir}")
+    return 0 if written else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())
--- a/scripts/run_citygml_to_glb.sh
+++ b/scripts/run_citygml_to_glb.sh
@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+# End-to-end helper to convert CityGML tiles to tile-local GLBs.
+# Assumes heightmaps/textures are already exported so tile_index.csv exists.
+
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+
+RAW_DIR="${RAW_DIR:-$ROOT/raw/citygml/lod2}"
+CITYJSON_DIR="${CITYJSON_DIR:-$ROOT/work/cityjson}"
+CLEAN_DIR="${CLEAN_DIR:-$ROOT/work/cityjson_clean}"
+TRI_DIR="${TRI_DIR:-$ROOT/work/cityjson_tri}"
+TRI_LOCAL_DIR="${TRI_LOCAL_DIR:-$ROOT/work/cityjson_tri_local}"
+SPLIT_DIR="${SPLIT_DIR:-$ROOT/work/cityjson_split_local}"
+GLB_DIR="${GLB_DIR:-$ROOT/export_unity/buildings_glb}"
+GLB_SPLIT_DIR="${GLB_SPLIT_DIR:-$ROOT/export_unity/buildings_glb_split}"
+TILE_INDEX="${TILE_INDEX:-$ROOT/export_unity/tile_index.csv}"
+
+CITYGML_TOOLS="${CITYGML_TOOLS:-$ROOT/tools/citygml-tools-2.4.0/citygml-tools}"
+CJIO_CMD="${CJIO:-uv run cjio}"
+PYTHON_CMD="${PYTHON_CMD:-uv run python}"
+
+log() {
+  printf '[citygml->glb] %s\n' "$*"
+}
+
+resolve_cityjson() {
+  local path="$1"
+  if [[ -f "$path" ]]; then
+    echo "$path"
+    return 0
+  fi
+  if [[ -d "$path" ]]; then
+    local base
+    base="$(basename "$path" .city.json)"
+    if [[ -f "$path/$base.json" ]]; then
+      echo "$path/$base.json"
+      return 0
+    fi
+    local first
+    first="$(find "$path" -maxdepth 1 -name '*.json' | head -n1 || true)"
+    if [[ -n "$first" ]]; then
+      echo "$first"
+      return 0
+    fi
+  fi
+  return 1
+}
+
+if [[ ! -f "$TILE_INDEX" ]]; then
+  echo "tile_index.csv missing at $TILE_INDEX; run heightmap export first." >&2
+  exit 1
+fi
+
+mapfile -t GMLS < <(find "$RAW_DIR" -maxdepth 1 -name 'LoD2_*.gml' | sort)
+if [[ ${#GMLS[@]} -eq 0 ]]; then
+  echo "No CityGML tiles found in $RAW_DIR" >&2
+  exit 1
+fi
+
+mkdir -p "$CITYJSON_DIR" "$CLEAN_DIR" "$TRI_DIR" "$TRI_LOCAL_DIR" "$SPLIT_DIR" "$GLB_DIR" "$GLB_SPLIT_DIR"
+
+log "CityGML -> CityJSON (${#GMLS[@]} tiles)"
+for gml in "${GMLS[@]}"; do
+  base="$(basename "$gml" .gml)"
+  out="$CITYJSON_DIR/${base}.city.json"
+  if [[ -e "$out" ]]; then
+    log "skip existing $out"
+    continue
+  fi
+  "$CITYGML_TOOLS" to-cityjson "$gml" -o "$out"
+done
+
+log "Clean invalid vertices"
+$PYTHON_CMD "$ROOT/scripts/clean_cityjson_vertices.py" --input-dir "$CITYJSON_DIR" --output-dir "$CLEAN_DIR"
+
+log "Triangulate"
+mapfile -t CITYJSONS < <(find "$CLEAN_DIR" -maxdepth 1 -name '*.city.json' | sort)
+for path in "${CITYJSONS[@]}"; do
+  base="$(basename "$path" .city.json)"
+  target="$TRI_DIR/${base}.tri.city.json"
+  if [[ -e "$target" ]]; then
+    log "skip existing $target"
+    continue
+  fi
+  input_json="$(resolve_cityjson "$path" || true)"
+  if [[ -z "$input_json" ]]; then
+    log "skip $path (cannot resolve json)"
+    continue
+  fi
+  $CJIO_CMD --ignore_duplicate_keys "$input_json" upgrade triangulate vertices_clean save "$target"
+done
+
+log "Rebase to tile-local XY"
+$PYTHON_CMD "$ROOT/scripts/rebase_cityjson_tiles.py" --input-dir "$TRI_DIR" --output-dir "$TRI_LOCAL_DIR" --tile-index "$TILE_INDEX"
+
+log "Split semantics (roof/wall)"
+$PYTHON_CMD "$ROOT/scripts/split_semantics.py" --input-dir "$TRI_LOCAL_DIR" --output-dir "$SPLIT_DIR"
+
+log "Export GLB (base)"
+mapfile -t TRIS < <(find "$TRI_LOCAL_DIR" -maxdepth 1 -name '*.tri.city.json' | sort)
+for tri in "${TRIS[@]}"; do
+  base="$(basename "$tri" .tri.city.json)"
+  out="$GLB_DIR/${base}.glb"
+  if [[ -e "$out" ]]; then
+    log "skip existing $out"
+    continue
+  fi
+  $CJIO_CMD "$tri" export glb "$out"
+done
+
+log "Export GLB (roof/wall)"
+mapfile -t ROOFS < <(find "$SPLIT_DIR" -maxdepth 1 -name '*.roof.city.json' | sort)
+for roof in "${ROOFS[@]}"; do
+  base="$(basename "$roof" .roof.city.json)"
+  out="$GLB_SPLIT_DIR/${base}_roof.glb"
+  if [[ -e "$out" ]]; then
+    log "skip existing $out"
+    continue
+  fi
+  $CJIO_CMD "$roof" export glb "$out"
+done
+
+mapfile -t WALLS < <(find "$SPLIT_DIR" -maxdepth 1 -name '*.wall.city.json' | sort)
+for wall in "${WALLS[@]}"; do
+  base="$(basename "$wall" .wall.city.json)"
+  out="$GLB_SPLIT_DIR/${base}_wall.glb"
+  if [[ -e "$out" ]]; then
+    log "skip existing $out"
+    continue
+  fi
+  $CJIO_CMD "$wall" export glb "$out"
+done
+
+log "Verify outputs"
+$PYTHON_CMD "$ROOT/scripts/verify_pipeline.py" --mode both --tri-dir "$TRI_LOCAL_DIR" --split-dir "$SPLIT_DIR"
+
+log "Done"