From c930d5f1e1e95c2963038b9006add80aa7552b79 Mon Sep 17 00:00:00 2001 From: "s0wlz (Matthias Puchstein)" Date: Fri, 23 Jan 2026 16:06:12 +0100 Subject: [PATCH] Add tile_key to manifest --- PIPELINE.md | 19 ++++++++++++++++--- README.md | 5 +++-- geodata_config.example.toml | 7 +++++++ geodata_config.toml | 7 +++++++ geodata_pipeline/config.py | 11 +++++++++++ geodata_pipeline/heightmaps.py | 20 ++++++++++++++++++-- 6 files changed, 62 insertions(+), 7 deletions(-) diff --git a/PIPELINE.md b/PIPELINE.md index 41f4639..a7096fc 100644 --- a/PIPELINE.md +++ b/PIPELINE.md @@ -47,7 +47,7 @@ Visual documentation showing how the GeoData pipeline transforms raw geospatial │ *.png (16-bit) │ *.jpg (2048²) │ *.glb │ *.csv │ │ *.pgw │ *.jgw │ │ trees_tiles/*.glb │ ├─────────────────┴─────────────────┴─────────────────┴───────────────────────┤ -│ tile_index.csv (manifest with bounds + elevation range) │ +│ tile_index.csv (manifest with bounds + elevation range + tile_key) │ ├─────────────────────────────────────────────────────────────────────────────┤ │ export_unity/ directory │ └────────────────────────────────────────┬────────────────────────────────────┘ @@ -77,7 +77,7 @@ Visual documentation showing how the GeoData pipeline transforms raw geospatial ▼ ┌─────────────────┐ │ HEIGHTMAP │◄──── MUST RUN FIRST - │ export_height │ Creates tile_index.csv + │ export_height │ Creates tile_index.csv (+tile_key) │ maps() │ Creates work/dgm.vrt └────────┬────────┘ │ @@ -208,7 +208,19 @@ TREES-ENHANCED: | `trees/*.csv` | CSV | - | Tree positions (x,y,z,h,r) | | `trees_tiles/*.glb` | glTF Binary | - | Proxy geometry chunks | | `street_furniture/*.csv` | CSV | - | Furniture positions | -| `tile_index.csv` | CSV | - | Manifest (bounds, min/max) | +| `tile_index.csv` | CSV | - | Manifest (bounds, min/max, tile_key) | + +--- + +## Tile Key Formula + +`tile_key` is appended to `tile_index.csv` during heightmap export: + +``` +tile_key = f"{floor((xmin + overlap_x) / tile_size_x)}_{floor((ymin + overlap_y) / tile_size_y)}" +``` + +Defaults (in `[tile_key]`): `tile_size_x=1000.0`, `tile_size_y=1000.0`, `overlap_x=0.5`, `overlap_y=0.5`, `enabled=true`. --- @@ -245,6 +257,7 @@ Config ├── export: ExportConfig # Output directories ├── heightmap: HeightmapConfig # out_res=1025, tile_size_m=1000 ├── ortho: OrthoConfig # out_res=2048, quality=90 +├── tile_key: TileKeyConfig # tile_size_x/y=1000, overlap_x/y=0.5 ├── buildings: BuildingConfig # triangle budget 200-350k ├── trees: TreeConfig # max_trees=5000, chunk_grid=4x4 ├── pointcloud: PointCloudConfig # LPG/LPO/BDOM directories diff --git a/README.md b/README.md index d9bf747..fb9a647 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ This repository converts DGM1 elevation tiles into Unity-ready 16-bit PNG height - `archive/` — offline storage for untouched downloads (e.g., zipped DOP/CityGML tiles, dop20 filelist). - `work/` — intermediates such as `dgm.vrt` and `_tmp.tif` files; safe to delete/regenerate. - `export_unity/height_png16/` — final 16-bit PNG heightmaps for Unity import. -- `export_unity/tile_index.csv` — manifest mapping tile IDs to world bounds and global min/max used for scaling; built during heightmap export and required by orthophotos. +- `export_unity/tile_index.csv` — manifest mapping tile IDs to world bounds and global min/max used for scaling, plus `tile_key`; built during heightmap export and required by orthophotos. - `export_unity/ortho_jpg/` — cropped orthophoto tiles aligned to the terrain grid (JPEG + worldfiles). - `geodata_to_unity.py` — main CLI (uses `geodata_pipeline/` library modules). - `scripts/` — helpers to create the directory tree and fetch DOP20 inputs. @@ -38,7 +38,7 @@ This repository converts DGM1 elevation tiles into Unity-ready 16-bit PNG height 4. Import the PNGs into Unity Terrains using `tile_index.csv` for placement and consistent height scaling (0–65535). ### How the export works -- Heightmaps: the pipeline builds `work/dgm.vrt` from all `raw/dgm1/*.tif`, computes a global min/max once, and warps each tile footprint to `heightmap.out_res` with `srcNodata=-9999` and `dstNodata` set to the global min. PNGs are scaled to `[0, 65535]` with worldfiles and listed in `export_unity/tile_index.csv`. +- Heightmaps: the pipeline builds `work/dgm.vrt` from all `raw/dgm1/*.tif`, computes a global min/max once, and warps each tile footprint to `heightmap.out_res` with `srcNodata=-9999` and `dstNodata` set to the global min. PNGs are scaled to `[0, 65535]` with worldfiles and listed in `export_unity/tile_index.csv` with `tile_key = f"{floor((xmin + overlap_x) / tile_size_x)}_{floor((ymin + overlap_y) / tile_size_y)}"` (defaults: `tile_size_x=1000.0`, `tile_size_y=1000.0`, `overlap_x=0.5`, `overlap_y=0.5` in `[tile_key]`). - Orthophotos: `work/dop.vrt` is built from `raw/dop20/jp2/*.jp2`; the manifest drives the cropping bounds. JPEG tiles are written to `export_unity/ortho_jpg/` with matching `.jgw` worldfiles. If the manifest is missing, the orthophoto export aborts—run the heightmap export first or use `--export all`. - Archives: `--build-from-archive` expands every `*.zip` under `archive/*` into the matching `raw/*` directories and copies `archive/dop20/filelist.txt` next to `raw/dop20/` for the downloader. - Cleanup: temporary `_tmp.tif` and GDAL aux XML files under `work/` and `raw/dgm1/` are removed at the end of the heightmap export; avoid storing non-GDAL metadata in those folders. @@ -53,6 +53,7 @@ This repository converts DGM1 elevation tiles into Unity-ready 16-bit PNG height ### Workflow Notes - The pipeline computes a global min/max from the VRT to scale all tiles consistently; adjust `heightmap.out_res` or `heightmap.resample` in `geodata_config.toml` if your AOI or target resolution changes. +- `tile_key` config controls the tile grouping key in the manifest; defaults are `tile_size_x=1000.0`, `tile_size_y=1000.0`, `overlap_x=0.5`, `overlap_y=0.5` with `enabled=true`. - `_tmp.tif` files in `work/` are transient; you can delete `work/` to force a clean rebuild. - Keep file names stable to avoid churn in Unity scenes; re-exports overwrite in place. - Large raw datasets are intentionally excluded from version control—document download sources or scripts instead of committing data. diff --git a/geodata_config.example.toml b/geodata_config.example.toml index b4b1422..5edb347 100644 --- a/geodata_config.example.toml +++ b/geodata_config.example.toml @@ -26,6 +26,13 @@ out_res = 1025 resample = "bilinear" tile_size_m = 1000 +[tile_key] +tile_size_x = 1000.0 +tile_size_y = 1000.0 +overlap_x = 0.5 +overlap_y = 0.5 +enabled = true + [ortho] out_res = 2048 jpeg_quality = 90 diff --git a/geodata_config.toml b/geodata_config.toml index 2a3b87b..2b5a8a1 100644 --- a/geodata_config.toml +++ b/geodata_config.toml @@ -25,6 +25,13 @@ out_res = 1025 resample = "bilinear" tile_size_m = 1000 +[tile_key] +tile_size_x = 1000.0 +tile_size_y = 1000.0 +overlap_x = 0.5 +overlap_y = 0.5 +enabled = true + [ortho] out_res = 2048 jpeg_quality = 90 diff --git a/geodata_pipeline/config.py b/geodata_pipeline/config.py index e56e26e..8d7da46 100644 --- a/geodata_pipeline/config.py +++ b/geodata_pipeline/config.py @@ -54,6 +54,15 @@ class OrthoConfig: jpeg_quality: int = 90 +@dataclass +class TileKeyConfig: + tile_size_x: float = 1000.0 + tile_size_y: float = 1000.0 + overlap_x: float = 0.5 + overlap_y: float = 0.5 + enabled: bool = True + + @dataclass class BuildingConfig: out_dir: str = "export_unity/buildings_tiles" @@ -151,6 +160,7 @@ class Config: export: ExportConfig = field(default_factory=ExportConfig) heightmap: HeightmapConfig = field(default_factory=HeightmapConfig) ortho: OrthoConfig = field(default_factory=OrthoConfig) + tile_key: TileKeyConfig = field(default_factory=TileKeyConfig) buildings: BuildingConfig = field(default_factory=BuildingConfig) trees: TreeConfig = field(default_factory=TreeConfig) # Enhanced pipeline configs @@ -179,6 +189,7 @@ class Config: export=ExportConfig(**_filter_kwargs(ExportConfig, data.get("export", {}))), heightmap=HeightmapConfig(**_filter_kwargs(HeightmapConfig, data.get("heightmap", {}))), ortho=OrthoConfig(**_filter_kwargs(OrthoConfig, data.get("ortho", {}))), + tile_key=TileKeyConfig(**_filter_kwargs(TileKeyConfig, data.get("tile_key", {}))), buildings=BuildingConfig(**_filter_kwargs(BuildingConfig, data.get("buildings", {}))), trees=TreeConfig(**_filter_kwargs(TreeConfig, data.get("trees", {}))), # Enhanced pipeline configs (with defaults for backward compat) diff --git a/geodata_pipeline/heightmaps.py b/geodata_pipeline/heightmaps.py index cf52035..4dc4e86 100644 --- a/geodata_pipeline/heightmaps.py +++ b/geodata_pipeline/heightmaps.py @@ -1,6 +1,7 @@ from __future__ import annotations import glob +import math import os from typing import Iterable @@ -34,8 +35,12 @@ def export_heightmaps(cfg: Config, *, force_vrt: bool = False) -> int: gmin, gmax = band.ComputeRasterMinMax(False) print(f"GLOBAL_MIN={gmin}, GLOBAL_MAX={gmax}") + tile_key_cfg = cfg.tile_key + tile_key_enabled = tile_key_cfg.enabled + tile_key_seen: set[str] = set() + with open(cfg.export.manifest_path, "w", encoding="utf-8") as f: - f.write("tile_id,xmin,ymin,xmax,ymax,global_min,global_max,out_res\n") + f.write("tile_id,xmin,ymin,xmax,ymax,global_min,global_max,out_res,tile_key\n") skipped = 0 written = 0 @@ -59,6 +64,15 @@ def export_heightmaps(cfg: Config, *, force_vrt: bool = False) -> int: base = os.path.splitext(os.path.basename(tif))[0] tile_id = base + tile_key = "" + if tile_key_enabled: + x_key = math.floor((xmin + tile_key_cfg.overlap_x) / tile_key_cfg.tile_size_x) + y_key = math.floor((ymin + tile_key_cfg.overlap_y) / tile_key_cfg.tile_size_y) + tile_key = f"{x_key}_{y_key}" + if tile_key in tile_key_seen: + print(f"Warning: duplicate tile_key {tile_key} for tile {tile_id}") + tile_key_seen.add(tile_key) + tmp_path = os.path.join(cfg.work.work_dir, f"{tile_id}_tmp.tif") out_path = os.path.join(cfg.export.heightmap_dir, f"{tile_id}.png") @@ -92,7 +106,9 @@ def export_heightmaps(cfg: Config, *, force_vrt: bool = False) -> int: safe_remove(tmp_path) safe_remove(f"{tmp_path}.aux.xml") - f.write(f"{tile_id},{xmin},{ymin},{xmax},{ymax},{gmin},{gmax},{cfg.heightmap.out_res}\n") + f.write( + f"{tile_id},{xmin},{ymin},{xmax},{ymax},{gmin},{gmax},{cfg.heightmap.out_res},{tile_key}\n" + ) print(f"Wrote {out_path}") written += 1