Add tile_key to manifest
This commit is contained in:
19
PIPELINE.md
19
PIPELINE.md
@@ -47,7 +47,7 @@ Visual documentation showing how the GeoData pipeline transforms raw geospatial
|
||||
│ *.png (16-bit) │ *.jpg (2048²) │ *.glb │ *.csv │
|
||||
│ *.pgw │ *.jgw │ │ trees_tiles/*.glb │
|
||||
├─────────────────┴─────────────────┴─────────────────┴───────────────────────┤
|
||||
│ tile_index.csv (manifest with bounds + elevation range) │
|
||||
│ tile_index.csv (manifest with bounds + elevation range + tile_key) │
|
||||
├─────────────────────────────────────────────────────────────────────────────┤
|
||||
│ export_unity/ directory │
|
||||
└────────────────────────────────────────┬────────────────────────────────────┘
|
||||
@@ -77,7 +77,7 @@ Visual documentation showing how the GeoData pipeline transforms raw geospatial
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ HEIGHTMAP │◄──── MUST RUN FIRST
|
||||
│ export_height │ Creates tile_index.csv
|
||||
│ export_height │ Creates tile_index.csv (+tile_key)
|
||||
│ maps() │ Creates work/dgm.vrt
|
||||
└────────┬────────┘
|
||||
│
|
||||
@@ -208,7 +208,19 @@ TREES-ENHANCED:
|
||||
| `trees/*.csv` | CSV | - | Tree positions (x,y,z,h,r) |
|
||||
| `trees_tiles/*.glb` | glTF Binary | - | Proxy geometry chunks |
|
||||
| `street_furniture/*.csv` | CSV | - | Furniture positions |
|
||||
| `tile_index.csv` | CSV | - | Manifest (bounds, min/max) |
|
||||
| `tile_index.csv` | CSV | - | Manifest (bounds, min/max, tile_key) |
|
||||
|
||||
---
|
||||
|
||||
## Tile Key Formula
|
||||
|
||||
`tile_key` is appended to `tile_index.csv` during heightmap export:
|
||||
|
||||
```
|
||||
tile_key = f"{floor((xmin + overlap_x) / tile_size_x)}_{floor((ymin + overlap_y) / tile_size_y)}"
|
||||
```
|
||||
|
||||
Defaults (in `[tile_key]`): `tile_size_x=1000.0`, `tile_size_y=1000.0`, `overlap_x=0.5`, `overlap_y=0.5`, `enabled=true`.
|
||||
|
||||
---
|
||||
|
||||
@@ -245,6 +257,7 @@ Config
|
||||
├── export: ExportConfig # Output directories
|
||||
├── heightmap: HeightmapConfig # out_res=1025, tile_size_m=1000
|
||||
├── ortho: OrthoConfig # out_res=2048, quality=90
|
||||
├── tile_key: TileKeyConfig # tile_size_x/y=1000, overlap_x/y=0.5
|
||||
├── buildings: BuildingConfig # triangle budget 200-350k
|
||||
├── trees: TreeConfig # max_trees=5000, chunk_grid=4x4
|
||||
├── pointcloud: PointCloudConfig # LPG/LPO/BDOM directories
|
||||
|
||||
@@ -17,7 +17,7 @@ This repository converts DGM1 elevation tiles into Unity-ready 16-bit PNG height
|
||||
- `archive/` — offline storage for untouched downloads (e.g., zipped DOP/CityGML tiles, dop20 filelist).
|
||||
- `work/` — intermediates such as `dgm.vrt` and `_tmp.tif` files; safe to delete/regenerate.
|
||||
- `export_unity/height_png16/` — final 16-bit PNG heightmaps for Unity import.
|
||||
- `export_unity/tile_index.csv` — manifest mapping tile IDs to world bounds and global min/max used for scaling; built during heightmap export and required by orthophotos.
|
||||
- `export_unity/tile_index.csv` — manifest mapping tile IDs to world bounds and global min/max used for scaling, plus `tile_key`; built during heightmap export and required by orthophotos.
|
||||
- `export_unity/ortho_jpg/` — cropped orthophoto tiles aligned to the terrain grid (JPEG + worldfiles).
|
||||
- `geodata_to_unity.py` — main CLI (uses `geodata_pipeline/` library modules).
|
||||
- `scripts/` — helpers to create the directory tree and fetch DOP20 inputs.
|
||||
@@ -38,7 +38,7 @@ This repository converts DGM1 elevation tiles into Unity-ready 16-bit PNG height
|
||||
4. Import the PNGs into Unity Terrains using `tile_index.csv` for placement and consistent height scaling (0–65535).
|
||||
|
||||
### How the export works
|
||||
- Heightmaps: the pipeline builds `work/dgm.vrt` from all `raw/dgm1/*.tif`, computes a global min/max once, and warps each tile footprint to `heightmap.out_res` with `srcNodata=-9999` and `dstNodata` set to the global min. PNGs are scaled to `[0, 65535]` with worldfiles and listed in `export_unity/tile_index.csv`.
|
||||
- Heightmaps: the pipeline builds `work/dgm.vrt` from all `raw/dgm1/*.tif`, computes a global min/max once, and warps each tile footprint to `heightmap.out_res` with `srcNodata=-9999` and `dstNodata` set to the global min. PNGs are scaled to `[0, 65535]` with worldfiles and listed in `export_unity/tile_index.csv` with `tile_key = f"{floor((xmin + overlap_x) / tile_size_x)}_{floor((ymin + overlap_y) / tile_size_y)}"` (defaults: `tile_size_x=1000.0`, `tile_size_y=1000.0`, `overlap_x=0.5`, `overlap_y=0.5` in `[tile_key]`).
|
||||
- Orthophotos: `work/dop.vrt` is built from `raw/dop20/jp2/*.jp2`; the manifest drives the cropping bounds. JPEG tiles are written to `export_unity/ortho_jpg/` with matching `.jgw` worldfiles. If the manifest is missing, the orthophoto export aborts—run the heightmap export first or use `--export all`.
|
||||
- Archives: `--build-from-archive` expands every `*.zip` under `archive/*` into the matching `raw/*` directories and copies `archive/dop20/filelist.txt` next to `raw/dop20/` for the downloader.
|
||||
- Cleanup: temporary `_tmp.tif` and GDAL aux XML files under `work/` and `raw/dgm1/` are removed at the end of the heightmap export; avoid storing non-GDAL metadata in those folders.
|
||||
@@ -53,6 +53,7 @@ This repository converts DGM1 elevation tiles into Unity-ready 16-bit PNG height
|
||||
|
||||
### Workflow Notes
|
||||
- The pipeline computes a global min/max from the VRT to scale all tiles consistently; adjust `heightmap.out_res` or `heightmap.resample` in `geodata_config.toml` if your AOI or target resolution changes.
|
||||
- `tile_key` config controls the tile grouping key in the manifest; defaults are `tile_size_x=1000.0`, `tile_size_y=1000.0`, `overlap_x=0.5`, `overlap_y=0.5` with `enabled=true`.
|
||||
- `_tmp.tif` files in `work/` are transient; you can delete `work/` to force a clean rebuild.
|
||||
- Keep file names stable to avoid churn in Unity scenes; re-exports overwrite in place.
|
||||
- Large raw datasets are intentionally excluded from version control—document download sources or scripts instead of committing data.
|
||||
|
||||
@@ -26,6 +26,13 @@ out_res = 1025
|
||||
resample = "bilinear"
|
||||
tile_size_m = 1000
|
||||
|
||||
[tile_key]
|
||||
tile_size_x = 1000.0
|
||||
tile_size_y = 1000.0
|
||||
overlap_x = 0.5
|
||||
overlap_y = 0.5
|
||||
enabled = true
|
||||
|
||||
[ortho]
|
||||
out_res = 2048
|
||||
jpeg_quality = 90
|
||||
|
||||
@@ -25,6 +25,13 @@ out_res = 1025
|
||||
resample = "bilinear"
|
||||
tile_size_m = 1000
|
||||
|
||||
[tile_key]
|
||||
tile_size_x = 1000.0
|
||||
tile_size_y = 1000.0
|
||||
overlap_x = 0.5
|
||||
overlap_y = 0.5
|
||||
enabled = true
|
||||
|
||||
[ortho]
|
||||
out_res = 2048
|
||||
jpeg_quality = 90
|
||||
|
||||
@@ -54,6 +54,15 @@ class OrthoConfig:
|
||||
jpeg_quality: int = 90
|
||||
|
||||
|
||||
@dataclass
|
||||
class TileKeyConfig:
|
||||
tile_size_x: float = 1000.0
|
||||
tile_size_y: float = 1000.0
|
||||
overlap_x: float = 0.5
|
||||
overlap_y: float = 0.5
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class BuildingConfig:
|
||||
out_dir: str = "export_unity/buildings_tiles"
|
||||
@@ -151,6 +160,7 @@ class Config:
|
||||
export: ExportConfig = field(default_factory=ExportConfig)
|
||||
heightmap: HeightmapConfig = field(default_factory=HeightmapConfig)
|
||||
ortho: OrthoConfig = field(default_factory=OrthoConfig)
|
||||
tile_key: TileKeyConfig = field(default_factory=TileKeyConfig)
|
||||
buildings: BuildingConfig = field(default_factory=BuildingConfig)
|
||||
trees: TreeConfig = field(default_factory=TreeConfig)
|
||||
# Enhanced pipeline configs
|
||||
@@ -179,6 +189,7 @@ class Config:
|
||||
export=ExportConfig(**_filter_kwargs(ExportConfig, data.get("export", {}))),
|
||||
heightmap=HeightmapConfig(**_filter_kwargs(HeightmapConfig, data.get("heightmap", {}))),
|
||||
ortho=OrthoConfig(**_filter_kwargs(OrthoConfig, data.get("ortho", {}))),
|
||||
tile_key=TileKeyConfig(**_filter_kwargs(TileKeyConfig, data.get("tile_key", {}))),
|
||||
buildings=BuildingConfig(**_filter_kwargs(BuildingConfig, data.get("buildings", {}))),
|
||||
trees=TreeConfig(**_filter_kwargs(TreeConfig, data.get("trees", {}))),
|
||||
# Enhanced pipeline configs (with defaults for backward compat)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import glob
|
||||
import math
|
||||
import os
|
||||
from typing import Iterable
|
||||
|
||||
@@ -34,8 +35,12 @@ def export_heightmaps(cfg: Config, *, force_vrt: bool = False) -> int:
|
||||
gmin, gmax = band.ComputeRasterMinMax(False)
|
||||
print(f"GLOBAL_MIN={gmin}, GLOBAL_MAX={gmax}")
|
||||
|
||||
tile_key_cfg = cfg.tile_key
|
||||
tile_key_enabled = tile_key_cfg.enabled
|
||||
tile_key_seen: set[str] = set()
|
||||
|
||||
with open(cfg.export.manifest_path, "w", encoding="utf-8") as f:
|
||||
f.write("tile_id,xmin,ymin,xmax,ymax,global_min,global_max,out_res\n")
|
||||
f.write("tile_id,xmin,ymin,xmax,ymax,global_min,global_max,out_res,tile_key\n")
|
||||
|
||||
skipped = 0
|
||||
written = 0
|
||||
@@ -59,6 +64,15 @@ def export_heightmaps(cfg: Config, *, force_vrt: bool = False) -> int:
|
||||
base = os.path.splitext(os.path.basename(tif))[0]
|
||||
tile_id = base
|
||||
|
||||
tile_key = ""
|
||||
if tile_key_enabled:
|
||||
x_key = math.floor((xmin + tile_key_cfg.overlap_x) / tile_key_cfg.tile_size_x)
|
||||
y_key = math.floor((ymin + tile_key_cfg.overlap_y) / tile_key_cfg.tile_size_y)
|
||||
tile_key = f"{x_key}_{y_key}"
|
||||
if tile_key in tile_key_seen:
|
||||
print(f"Warning: duplicate tile_key {tile_key} for tile {tile_id}")
|
||||
tile_key_seen.add(tile_key)
|
||||
|
||||
tmp_path = os.path.join(cfg.work.work_dir, f"{tile_id}_tmp.tif")
|
||||
out_path = os.path.join(cfg.export.heightmap_dir, f"{tile_id}.png")
|
||||
|
||||
@@ -92,7 +106,9 @@ def export_heightmaps(cfg: Config, *, force_vrt: bool = False) -> int:
|
||||
safe_remove(tmp_path)
|
||||
safe_remove(f"{tmp_path}.aux.xml")
|
||||
|
||||
f.write(f"{tile_id},{xmin},{ymin},{xmax},{ymax},{gmin},{gmax},{cfg.heightmap.out_res}\n")
|
||||
f.write(
|
||||
f"{tile_id},{xmin},{ymin},{xmax},{ymax},{gmin},{gmax},{cfg.heightmap.out_res},{tile_key}\n"
|
||||
)
|
||||
print(f"Wrote {out_path}")
|
||||
written += 1
|
||||
|
||||
|
||||
Reference in New Issue
Block a user