From 83ffa457dd7df197e73aec133875c8020588afae Mon Sep 17 00:00:00 2001 From: "s0wlz (Matthias Puchstein)" Date: Sat, 13 Dec 2025 22:59:10 +0100 Subject: [PATCH] Add GIS-friendly exports and update contributor guide --- AGENTS.md | 7 +++- export_heightmaps.py | 84 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 78 insertions(+), 13 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2154553..aeb3f0c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,8 +2,11 @@ ## Project Structure & Module Organization - `raw_dgm1/`: source DGM1 tiles (`.tif` + `.tfw`) named `dgm1___.tif`. -- `work/`: transient artifacts such as `dgm.vrt` and intermediate `_tmp.tif` files; safe to regenerate. +- `raw_3dgeb_lod1/` and `raw_3dgeb_lod2/`: LoD1/LoD2 building geometries as `.gml` with sidecar `.txt` metadata. +- `raw_dop/`: orthophoto download artifacts (scripts, certs, `jp2`, `xml`, `j2w`); treat as read-only inputs. - `export_unity/`: delivery assets; `height_png16/` holds 16-bit heightmaps, `tile_index.csv` maps tile IDs to world bounds and global min/max. +- `work/`: transient artifacts such as `dgm.vrt` and intermediate `_tmp.tif` files; safe to regenerate. +- `3dgeblod1/`, `3dgeblod2/`, `dgm/`: preserved ZIPs and object lists documenting raw downloads; keep for provenance. - `export_heightmaps.py`: main Python pipeline that mosaics the VRT, rescales to UInt16, and writes Unity-friendly tiles. ## Build, Test, and Development Commands @@ -14,6 +17,7 @@ - Inspect a tile for sanity (optional spot check): `gdalinfo export_unity/height_png16/dgm1_32_328_5511.png | head` - Dependencies: GDAL with Python bindings (`osgeo`) available on PATH/PYTHONPATH. +- If processing LoD GML or DOP imagery, keep scripts/data under the respective `raw_*` folders; avoid scattering inputs. ## Coding Style & Naming Conventions - Python: 4-space indentation, snake_case identifiers, and module-level constants for tunables (e.g., `TILE_SIZE_M`, `OUT_RES`). @@ -30,3 +34,4 @@ - PRs should summarize scope, list commands executed (`gdalbuildvrt`, `export_heightmaps.py`), and call out data changes or new dependencies. - Avoid committing large raw datasets if not required; prefer documenting download steps or using `.gitignore` for temporary exports. - Include screenshots or brief notes when changes affect Unity import workflows or manifest formats. +- Note the presence of archived downloads in `3dgeblod*` and `dgm/`; do not delete them without replacing provenance elsewhere. diff --git a/export_heightmaps.py b/export_heightmaps.py index ff11c17..0a6320c 100644 --- a/export_heightmaps.py +++ b/export_heightmaps.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import glob -import math import os from osgeo import gdal @@ -16,26 +15,68 @@ RESAMPLE = "bilinear" os.makedirs("work", exist_ok=True) os.makedirs(OUT_DIR, exist_ok=True) -# Open VRT -ds = gdal.Open(VRT_PATH) -if ds is None: - raise SystemExit(f"Could not open {VRT_PATH}. Did you run gdalbuildvrt?") +gdal.UseExceptions() + + +def open_dataset(path, purpose): + """Open a dataset and fail fast with context.""" + try: + ds = gdal.Open(path) + except RuntimeError as exc: + raise SystemExit(f"{purpose}: {exc}") from exc + if ds is None: + raise SystemExit(f"{purpose}: GDAL returned None for {path}") + return ds + + +def safe_remove(path): + """Remove a file if present; return True when deleted.""" + try: + os.remove(path) + return True + except FileNotFoundError: + return False + except OSError as exc: + print(f"Warning: could not remove {path}: {exc}") + return False + + +def cleanup_aux_files(): + """Clear GDAL sidecars and leftover temp files to keep the repo tidy.""" + patterns = [ + os.path.join("work", "*_tmp.tif"), + os.path.join("work", "*_tmp.tif.aux.xml"), + os.path.join("work", "*.aux.xml"), + os.path.join(RAW_DIR, "*.aux.xml"), + ] + removed = 0 + for pattern in patterns: + for path in glob.glob(pattern): + if safe_remove(path): + removed += 1 + print(f"Cleanup removed {removed} temporary files/sidecars.") + + +ds = open_dataset(VRT_PATH, f"Could not open {VRT_PATH}. Did you run gdalbuildvrt?") band = ds.GetRasterBand(1) -# Strategy B: compute global min/max for CURRENT downloaded AOI (full scan for stability) gmin, gmax = band.ComputeRasterMinMax(False) print(f"GLOBAL_MIN={gmin}, GLOBAL_MAX={gmax}") -# Export manifest for Unity placement later manifest_path = os.path.join("export_unity", "tile_index.csv") with open(manifest_path, "w", encoding="utf-8") as f: f.write("tile_id,xmin,ymin,xmax,ymax,global_min,global_max,out_res\n") + skipped = 0 + written = 0 + for tif in sorted(glob.glob(os.path.join(RAW_DIR, "*.tif"))): - tds = gdal.Open(tif) - if tds is None: - print(f"Skipping unreadable: {tif}") + try: + tds = open_dataset(tif, f"Skipping unreadable {tif}") + except SystemExit as exc: + print(exc) + skipped += 1 continue gt = tds.GetGeoTransform() @@ -61,17 +102,36 @@ with open(manifest_path, "w", encoding="utf-8") as f: srcNodata=-9999, dstNodata=gmin, # fill nodata with global min to avoid deep pits ) - gdal.Warp(tmp_path, ds, options=warp_opts) + try: + gdal.Warp(tmp_path, ds, options=warp_opts) + except RuntimeError as exc: + print(f"Warp failed for {tile_id}: {exc}") + skipped += 1 + continue # Scale to UInt16 (0..65535) using Strategy-B global min/max trans_opts = gdal.TranslateOptions( outputType=gdal.GDT_UInt16, scaleParams=[(gmin, gmax, 0, 65535)], format="PNG", + creationOptions=["WORLDFILE=YES"], # emit .wld so GIS tools place tiles correctly ) - gdal.Translate(out_path, tmp_path, options=trans_opts) + try: + gdal.Translate(out_path, tmp_path, options=trans_opts) + except RuntimeError as exc: + print(f"Translate failed for {tile_id}: {exc}") + skipped += 1 + continue + safe_remove(tmp_path) + safe_remove(f"{tmp_path}.aux.xml") f.write(f"{tile_id},{xmin},{ymin},{xmax},{ymax},{gmin},{gmax},{OUT_RES}\n") print(f"Wrote {out_path}") + written += 1 print(f"Manifest: {manifest_path}") +print(f"Summary: wrote {written} tiles; skipped {skipped}.") +cleanup_aux_files() + +if skipped: + raise SystemExit(1)